chore: Wprazuch/nano v3 repro refinements (#564)

Signed-off-by: Wojciech Prazuch <[email protected]>

Author: wprazuch

packages/nemo-evaluator-launcher/examples/nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml

@@ -28,7 +28,7 @@ execution:
 
 target:
   api_endpoint:
-    model_id: nvidia/nemotron-nano-3-30b-a3b
+    model_id: nvidia/nemotron-3-nano-30b-a3b
     url: https://integrate.api.nvidia.com/v1/chat/completions
     api_key_name: NGC_API_KEY # API Key with access to build.nvidia.com
 
@@ -43,7 +43,7 @@ evaluation:
         max_new_tokens: 131072
         temperature: 0.99999
         top_p: 0.99999
-        parallelism: 512
+        parallelism: 1
         request_timeout: 3600
         max_retries: 10
         extra:
@@ -60,15 +60,18 @@ evaluation:
           use_response_logging: true
           max_logged_responses: 10
   tasks:
+  - name: ns_ifbench
+    nemo_evaluator_config:
+      config:
+        params:
+          extra:
+            num_repeats: 8
   - name: ns_bfcl_v3
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           temperature: 0.6
           top_p: 0.95
-          parallelism: 32
           extra:
             num_repeats: 1
             args: ++use_client_parsing=False
@@ -77,76 +80,43 @@ evaluation:
           adapter_config:
             use_caching: false
   - name: ns_bfcl_v4
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           max_new_tokens: 8192
-          parallelism: 128
           temperature: 0.6
           top_p: 0.95
           extra:
             num_repeats: 1
             args: ++use_client_parsing=False
   - name: ns_livecodebench
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           extra:
             num_repeats: 8
             dataset_split: test_v5_2407_2412
   - name: ns_mmlu_pro
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           extra:
             num_repeats: 1
             args: "++prompt_config=eval/aai/mcq-10choices-boxed"
   - name: ns_gpqa
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           extra:
             num_repeats: 8
             args: "++prompt_config=eval/aai/mcq-4choices"
-  - name: ns_aime2025
-    env_vars:
-      HF_TOKEN: HF_TOKEN
-      JUDGE_API_KEY: JUDGE_API_KEY
-    nemo_evaluator_config:
-      config:
-        params:
-          extra:
-            num_repeats: 64
-            args: ++prompt_config=/prompt_templates/math-oai.yaml
   - name: ns_scicode
-    env_vars:
-      HF_TOKEN: HF_TOKEN
-      JUDGE_API_KEY: JUDGE_API_KEY
-    nemo_evaluator_config:
-      config:
-        params:
-          extra:
-            num_repeats: 8
-  - name: ns_ifbench
-    env_vars:
-      HF_TOKEN: HF_TOKEN
     nemo_evaluator_config:
       config:
         params:
           extra:
             num_repeats: 8
   - name: ns_hle
-    env_vars:
-      HF_TOKEN: HF_TOKEN
-      JUDGE_API_KEY: JUDGE_API_KEY
     nemo_evaluator_config:
       config:
         params:
@@ -156,5 +126,5 @@ evaluation:
             judge:
               parallelism: 16
               model_id: openai/gpt-4o
-              url: <OPENAI_API_URL_FOR_JUDGE>
+              url: ??? # Set your OpenAI-compatible API URL for the judge model
               api_key: JUDGE_API_KEY

packages/nemo-evaluator-launcher/examples/nemotron/nano-v3-reproducibility.md

@@ -1,6 +1,6 @@
 # NVIDIA Nemotron 3 Nano 30B A3B — Reproducing Model Card Evaluation Results
 
-This tutorial demonstrates how to reproduce the evaluation results for the **NVIDIA Nemotron 3 Nano 30B A3B** model using the NeMo Evaluator Launcher.
+This tutorial demonstrates how to reproduce the evaluation results for the [**NVIDIA Nemotron 3 Nano 30B A3B**](https://build.nvidia.com/nvidia/nemotron-3-nano-30b-a3b) model using the NeMo Evaluator Launcher.
 
 ## Overview
 
@@ -33,7 +33,8 @@ pip install nemo-evaluator-launcher
 Or install from source:
 
 ```bash
-cd packages/nemo-evaluator-launcher
+git clone https://github.com/NVIDIA-NeMo/Evaluator.git
+cd Evaluator/packages/nemo-evaluator-launcher
 pip install -e .
 ```
 
@@ -66,28 +67,31 @@ export HF_HOME="/path/to/your/huggingface/cache"
 
 ## Running the Full Evaluation Suite
 
-### Option 1: Using the Config File Directly
+### 1. Get the Configuration
 
-Navigate to the examples directory and run:
+Clone the repository or download the [example config file](https://github.com/NVIDIA-NeMo/Evaluator/blob/main/packages/nemo-evaluator-launcher/examples/nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml):
 
 ```bash
-cd packages/nemo-evaluator-launcher/examples/nemotron
-
-nemo-evaluator-launcher run --config local_nvidia_nemotron_3_nano_30b_a3b.yaml
+git clone https://github.com/NVIDIA-NeMo/Evaluator.git
+cd Evaluator  # or navigate to where you placed the config file
 ```
 
-### Option 2: Using Config Path from Anywhere
+### 2. Run the Evaluation
 
 ```bash
-nemo-evaluator-launcher run --config /path/to/examples/nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml
+nemo-evaluator-launcher run \
+  --config packages/nemo-evaluator-launcher/examples/nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml
+# Or point to your path if you placed the file under a different location
 ```
 
-### Option 3: Dry Run (Preview Configuration)
+### 3. Dry Run (Preview Configuration)
 
 To preview the configuration without running the evaluation:
 
 ```bash
-nemo-evaluator-launcher run --config nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml --dry-run
+nemo-evaluator-launcher run \
+  --config packages/nemo-evaluator-launcher/examples/nemotron/local_nvidia_nemotron_3_nano_30b_a3b.yaml \
+  --dry-run
 ```
 
 ---
@@ -132,7 +136,7 @@ The evaluation uses the NVIDIA API endpoint:
 ```yaml
 target:
   api_endpoint:
-    model_id: nvidia/nemotron-nano-3-30b-a3b
+    model_id: nvidia/nemotron-3-nano-30b-a3b
     url: https://integrate.api.nvidia.com/v1/chat/completions
     api_key_name: NGC_API_KEY
 ```