Add network-wide grafana error links for Kubernetes CI (#2117)

* Add network-wide grafana error links for Kubernetes CI

* Prettier fix

Author: mchristou

javascript/packages/orchestrator/src/network.ts

@@ -2,6 +2,7 @@ import {
   CreateLogTable,
   TimeoutAbortController,
   decorators,
+  getLokiUrlForNetworkErrors,
 } from "@zombienet/utils";
 import fs from "fs";
 import {
@@ -340,6 +341,33 @@ export class Network {
         this.showNodeInfo(node, provider, logTable);
       }
     }
+
+    // Add network-wide error logs link for kubernetes provider
+    if (this.client.providerName === "kubernetes" && this.networkStartTime) {
+      const inCI = process.env.RUN_IN_CONTAINER === "1";
+      if (inCI) {
+        const networkLokiUrl = getLokiUrlForNetworkErrors(
+          this.namespace,
+          this.networkStartTime,
+        );
+        logTable.pushTo([
+          [
+            {
+              colSpan: 2,
+              hAlign: "center",
+              content: decorators.cyan("🌐 All nodes logs (Grafana)"),
+            },
+          ],
+          [
+            {
+              colSpan: 2,
+              content: decorators.bright(networkLokiUrl),
+            },
+          ],
+        ]);
+      }
+    }
+
     logTable.print();
   }
 

javascript/packages/orchestrator/src/test-runner/index.ts

@@ -1,6 +1,7 @@
 import {
   decorators,
   getLokiUrl,
+  getLokiUrlForNetworkErrors,
   readNetworkConfig,
   setLogType,
   sleep,
@@ -159,6 +160,22 @@ export async function run(
             console.log(
               `\n\n\t${decorators.red("❌ One or more of your test failed...")}`,
             );
+
+            // Show network-wide error logs link for kubernetes in CI when tests fail
+            if (network.client.providerName === "kubernetes" && inCI) {
+              const networkEndtime = new Date().getTime();
+              const networkLokiUrl = getLokiUrlForNetworkErrors(
+                network.namespace,
+                network.networkStartTime!,
+                networkEndtime,
+              );
+              console.log(
+                `\n\t${decorators.red("🔍 View error logs for all nodes in the network:")}`,
+              );
+              console.log(
+                `\t${decorators.bright(decorators.red(networkLokiUrl))}`,
+              );
+            }
           } else {
             success = true;
           }
@@ -220,6 +237,18 @@ export async function run(
                   }
                 }
 
+                // Add network-wide errors logs link
+                const networkLokiUrl = getLokiUrlForNetworkErrors(
+                  network.namespace,
+                  network.networkStartTime!,
+                  networkEndtime,
+                );
+                console.log(
+                  `\n\t${decorators.cyan("🌐 All nodes (relaychain + parachains) error logs:")} ${decorators.bright(
+                    networkLokiUrl,
+                  )}`,
+                );
+
                 // logs are also collected as artifacts
                 console.log(
                   `\n\n\t ${decorators.yellow(

javascript/packages/utils/src/constants.ts

@@ -1,2 +1,5 @@
 export const LOKI_URL_FOR_NODE =
   "https://grafana.teleport.parity.io/explore?orgId=1&left=%7B%22datasource%22:%22PCF9DACBDF30E12B3%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22PCF9DACBDF30E12B3%22%7D,%22editorMode%22:%22code%22,%22expr%22:%22%7Bnamespace%3D%5C%22{{namespace}}%5C%22,pod%3D%5C%22{{podName}}%5C%22%7D%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22{{from}}%22,%22to%22:%22{{to}}%22%7D%7D";
+
+export const LOKI_URL_FOR_NETWORK_ERRORS =
+  "https://grafana.teleport.parity.io/explore?orgId=1&left=%7B%22datasource%22:%22PCF9DACBDF30E12B3%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22PCF9DACBDF30E12B3%22%7D,%22editorMode%22:%22code%22,%22expr%22:%22%7Bnamespace%3D%5C%22{{namespace}}%5C%22%7D%20%7C~%20%5C%22(%3F%3Ai)error%7Cfail%7Cpanic%7Cfatal%5C%22%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22{{from}}%22,%22to%22:%22{{to}}%22%7D%7D";

javascript/packages/utils/src/misc.ts

@@ -1,6 +1,6 @@
 import { createHash, randomBytes } from "crypto";
 import { format } from "util";
-import { LOKI_URL_FOR_NODE } from "./constants";
+import { LOKI_URL_FOR_NODE, LOKI_URL_FOR_NETWORK_ERRORS } from "./constants";
 
 export async function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -120,6 +120,21 @@ export function getLokiUrl(
   return loki_url;
 }
 
+export function getLokiUrlForNetworkErrors(
+  namespace: string,
+  from: number | string,
+  to?: number | string,
+): string {
+  const loki_url = LOKI_URL_FOR_NETWORK_ERRORS.replace(
+    /{{namespace}}/,
+    namespace,
+  )
+    .replace(/{{from}}/, from.toString())
+    .replace(/{{to}}/, to?.toString() || "now");
+
+  return loki_url;
+}
+
 export const TimeoutAbortController = (time: number) => {
   const controller = new AbortController();
   setTimeout(() => controller.abort(), time * 1000);