feat: Adding qualificator util

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "dc05b0d5d14def9120f2bdfa82aec1fac3b5e388003989fd56263827f1fab85b",
+  "checksum": "0a147d75b6943fe7fce1c7d185a8c1576d83eebf42e9e15a9f046cae1ff72dbb",
   "crates": {
     "actix-codec 0.5.2": {
       "name": "actix-codec",
@@ -28565,35 +28565,29 @@
         ],
         "crate_features": {
           "common": [
+            "elf",
+            "errno",
             "general",
             "ioctl",
             "no_std"
           ],
           "selects": {
             "aarch64-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "arm-unknown-linux-gnueabi": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "armv7-unknown-linux-gnueabi": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
             ],
             "i686-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
@@ -28609,8 +28603,6 @@
               "system"
             ],
             "x86_64-unknown-linux-gnu": [
-              "elf",
-              "errno",
               "prctl",
               "std",
               "system"
@@ -35831,6 +35823,78 @@
       },
       "license": "MIT"
     },
+    "qualifier 0.4.3": {
+      "name": "qualifier",
+      "version": "0.4.3",
+      "repository": null,
+      "targets": [],
+      "library_target_name": null,
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "anyhow 1.0.86",
+              "target": "anyhow"
+            },
+            {
+              "id": "clap 4.5.9",
+              "target": "clap"
+            },
+            {
+              "id": "dirs 5.0.1",
+              "target": "dirs"
+            },
+            {
+              "id": "ic-nervous-system-common-test-keys 0.9.0",
+              "target": "ic_nervous_system_common_test_keys"
+            },
+            {
+              "id": "itertools 0.13.0",
+              "target": "itertools"
+            },
+            {
+              "id": "log 0.4.22",
+              "target": "log"
+            },
+            {
+              "id": "pretty_env_logger 0.5.0",
+              "target": "pretty_env_logger"
+            },
+            {
+              "id": "reqwest 0.12.5",
+              "target": "reqwest"
+            },
+            {
+              "id": "serde 1.0.204",
+              "target": "serde"
+            },
+            {
+              "id": "serde_json 1.0.120",
+              "target": "serde_json"
+            },
+            {
+              "id": "tokio 1.38.1",
+              "target": "tokio"
+            },
+            {
+              "id": "tokio-util 0.7.11",
+              "target": "tokio_util"
+            },
+            {
+              "id": "url 2.5.2",
+              "target": "url"
+            }
+          ],
+          "selects": {}
+        },
+        "edition": "2021",
+        "version": "0.4.3"
+      },
+      "license": null
+    },
     "quick-xml 0.23.1": {
       "name": "quick-xml",
       "version": "0.23.1",
@@ -50321,6 +50385,7 @@
     "np-notifications 0.4.3": "rs/np-notifications",
     "obs-canister-clients 0.4.3": "rs/ic-observability/obs-canister-clients",
     "prometheus-config-updater 0.4.3": "rs/ic-observability/prometheus-config-updater",
+    "qualifier 0.4.3": "rs/qualifier",
     "rollout-controller 0.4.3": "rs/rollout-controller",
     "service-discovery 0.4.3": "rs/ic-observability/service-discovery",
     "slack-notifications 0.4.3": "rs/slack-notifications",

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "rs/rollout-controller",
     "rs/slack-notifications",
     "rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics",
+    "rs/qualifier",
     "rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-types",
 ]
 
@@ -109,6 +110,7 @@ ic-base-types = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175
 ic-canister-client = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d" }
 ic-canister-client-sender = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d" }
 ic-canisters = { path = "rs/ic-canisters" }
+ic-nervous-system-common-test-keys = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d" }
 ic-cdk = "0.15.0"
 ic-config = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d" }
 ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dfinity/ic.git", rev = "5ba1412f9175d987661ae3c0d8dbd1ac3e092b7d" }

bazel/external_crates.bzl

@@ -66,6 +66,7 @@ def external_crates_repository():
             "//rs/slack-notifications:Cargo.toml",
             "//rs/rollout-controller:Cargo.toml",
             "//rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics:Cargo.toml",
+            "//rs/qualifier:Cargo.toml",
             "//rs/dre-canisters/trustworthy-node-metrics/src/trustworthy-node-metrics-types:Cargo.toml",
         ],
         splicing_config = splicing_config(

rs/cli/src/commands/mod.rs

@@ -28,14 +28,14 @@ mod api_boundary_nodes;
 mod completions;
 mod der_to_principal;
 mod firewall;
-mod get;
+pub mod get;
 mod heal;
 pub mod hostos;
 mod node_metrics;
 mod nodes;
 mod proposals;
 mod propose;
-mod qualify;
+pub mod qualify;
 mod registry;
 mod subnet;
 mod update_unassigned_nodes;
@@ -170,7 +170,7 @@ pub trait ExecutableCommand {
 
     fn validate(&self, cmd: &mut Command);
 
-    async fn execute(&self, ctx: DreContext) -> anyhow::Result<()>;
+    fn execute(&self, ctx: DreContext) -> impl std::future::Future<Output = anyhow::Result<()>>;
 
     fn validate_min_nakamoto_coefficients(cmd: &mut clap::Command, min_nakamoto_coefficients: &[String]) {
         let _ = Self::_parse_min_nakamoto_coefficients_inner(Some(cmd), min_nakamoto_coefficients);

rs/cli/src/commands/qualify/execute.rs

@@ -12,26 +12,26 @@ use crate::{
 pub struct Execute {
     /// Version which is to be qualified
     #[clap(long, short)]
-    version: String,
+    pub version: String,
 
     /// Starting version for the network.
     ///
     /// If left empty, the tool will use the current NNS version
     #[clap(long, short)]
-    from_version: Option<String>,
+    pub from_version: Option<String>,
 
     /// Specify the steps to run
     /// A range can be: `4`, `3..`, `..3, `1..3`
     #[clap(long)]
-    step_range: Option<String>,
+    pub step_range: Option<String>,
 
     /// Name of the deployment used for prometheus querying of `ic` label: `staging`, `from-config`...
     #[clap(long)]
-    deployment_name: String,
+    pub deployment_name: String,
 
     /// Prometheus compliant endpoint
     #[clap(long)]
-    prometheus_endpoint: String,
+    pub prometheus_endpoint: String,
 }
 
 impl ExecutableCommand for Execute {

rs/cli/src/commands/qualify/mod.rs

@@ -4,17 +4,17 @@ use list::List;
 
 use super::ExecutableCommand;
 
-mod execute;
+pub mod execute;
 mod list;
 
 #[derive(Args, Debug)]
 pub struct QualifyCmd {
     #[clap(subcommand)]
-    subcommand: QualifyCommands,
+    pub subcommand: QualifyCommands,
 }
 
 #[derive(Subcommand, Debug)]
-enum QualifyCommands {
+pub enum QualifyCommands {
     /// List all steps present in the qualification
     List(List),
     /// Execute the qualification

rs/cli/src/lib.rs

@@ -1,3 +1,9 @@
 #![allow(dead_code)]
 pub mod auth;
+pub mod commands;
+pub mod ctx;
 pub mod ic_admin;
+mod operations;
+mod qualification;
+mod runner;
+mod subnet_manager;

rs/cli/src/qualification/mod.rs

@@ -287,14 +287,22 @@ impl QualificationExecutor {
             }
             print_text(format!("Executing step {}: `{}`", ordered_step.index, ordered_step.step.name()));
 
-            ordered_step.step.execute(&self.dre_ctx).await?;
+            let step_future = || async { ordered_step.step.execute(&self.dre_ctx).await };
+            step_future.retry(&ExponentialBuilder::default()).await?;
 
             print_text(format!("Executed step {}: `{}`", ordered_step.index, ordered_step.step.name()));
 
             let registry = self.dre_ctx.registry().await;
             print_text(format!("Syncing with registry after step {}", ordered_step.index));
             let sync_registry = || async { registry.sync_with_nns().await };
-            sync_registry.retry(&ExponentialBuilder::default()).await?;
+            // If the system subnet downgraded it could be some time until it boots up
+            sync_registry
+                .retry(
+                    &ExponentialBuilder::default()
+                        .with_max_times(10)
+                        .with_max_delay(Duration::from_secs(5 * 60)),
+                )
+                .await?;
         }
 
         print_text(format!("Qualification of {} finished successfully!", self.to_version));

rs/cli/src/qualification/run_workload_test.rs

@@ -99,16 +99,13 @@ async fn ensure_finalization_rate_for_subnet(
         common_labels
     );
     let query = format!("avg(rate({}[{}s]))", query_selector, duration);
-    print_text(format!("Running query: {}", query));
-    let response = client
+    let request = client
         .get(prom_endpoint)
         .header("Accept", "application/json")
-        .query(&[("time", end_timestamp.to_string()), ("query", query)])
-        .send()
-        .await?
-        .error_for_status()?
-        .json::<Value>()
-        .await?;
+        .query(&[("time", end_timestamp.to_string()), ("query", query)]);
+    print_text(format!("Running query: {:?}", request));
+    let response = request.send().await?.error_for_status()?.json::<Value>().await?;
+    print_text(format!("Received response: \n{}", serde_json::to_string_pretty(&response)?));
 
     let finalization_rate = response["data"]["result"][0]["value"][1]
         .as_str()

rs/qualifier/BUILD.bazel

@@ -0,0 +1,23 @@
+load("@crate_index_dre//:defs.bzl", "aliases", "all_crate_deps")
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test")
+load("@//rs:oci_images.bzl", "rust_binary_oci_image_rules")
+
+DEPS = [
+    "//rs/cli:dre-lib",
+    "//rs/ic-canisters",
+    "//rs/ic-management-types",
+    "//rs/ic-management-backend:ic-management-backend-lib"
+]
+
+rust_binary(
+    name = "qualifier",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    stamp = 1,
+    deps = all_crate_deps(
+        normal = True,
+    ) + DEPS,
+)

rs/qualifier/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "qualifier"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+description.workspace = true
+documentation.workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }
+tokio = { workspace = true }
+log = { workspace = true }
+pretty_env_logger = { workspace = true }
+ic-nervous-system-common-test-keys = { workspace = true }
+dirs = { workspace = true }
+dre = { path = "../cli" }
+ic-management-types = { path = "../ic-management-types" }
+ic-canisters = { path = "../ic-canisters" }
+serde_json = { workspace = true }
+serde = { workspace = true }
+tokio-util = { workspace = true }
+itertools = { workspace = true }
+reqwest = { workspace = true }
+url = { workspace = true }
+ic-management-backend = { path = "../ic-management-backend" }