Skip to content

Commit dec29ed

Browse files
xinhe-nvLarryXFly
authored and
Ria Jain
committed
tests: add TestNemotronH cuda graph tests (NVIDIA#6390)
Signed-off-by: Xin He (SW-GPU) <[email protected]> Co-authored-by: Larry <[email protected]>
1 parent 46ea9e6 commit dec29ed

File tree

4 files changed

+114
-7
lines changed

4 files changed

+114
-7
lines changed

‎tests/integration/defs/accuracy/references/gsm8k.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,16 @@ nvidia/Nemotron-H-8B-Base-8K:
9696
- quant_algo: FP8
9797
kv_cache_quant_algo: FP8
9898
accuracy: 85.78
99+
nvidia/Nemotron-H-47B-Base-8K:
100+
- accuracy: 88.82
101+
- quant_algo: FP8
102+
kv_cache_quant_algo: FP8
103+
accuracy: 88.55
104+
nvidia/Nemotron-H-56B-Base-8K:
105+
- accuracy: 89.27
106+
- quant_algo: FP8
107+
kv_cache_quant_algo: FP8
108+
accuracy: 89.27
99109
nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
100110
- accuracy: 37.15
101111
- quant_algo: FP8

‎tests/integration/defs/accuracy/references/mmlu.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,16 @@ nvidia/Nemotron-H-8B-Base-8K:
185185
- quant_algo: FP8
186186
kv_cache_quant_algo: FP8
187187
accuracy: 69.180
188+
nvidia/Nemotron-H-47B-Base-8K:
189+
- accuracy: 83.26
190+
- quant_algo: FP8
191+
kv_cache_quant_algo: FP8
192+
accuracy: 82.68
193+
nvidia/Nemotron-H-56B-Base-8K:
194+
- accuracy: 83.82
195+
- quant_algo: FP8
196+
kv_cache_quant_algo: FP8
197+
accuracy: 83.82
188198
microsoft/Phi-4-mini-instruct:
189199
- accuracy: 68.98
190200
# Created a dummy accuracy to track tp_size=2 for phi4-mini model.

‎tests/integration/defs/accuracy/test_llm_api_pytorch.py

Lines changed: 88 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1602,31 +1602,114 @@ class TestNemotronH(LlmapiAccuracyTestHarness):
16021602
MODEL_NAME = "nvidia/Nemotron-H-8B-Base-8K"
16031603
MODEL_PATH = f"{llm_models_root()}/Nemotron-H-8B-Base-8K"
16041604

1605-
def test_auto_dtype(self):
1605+
@parametrize_with_ids("cuda_graph", [False, True])
1606+
def test_auto_dtype(self, cuda_graph):
16061607
# TODO: remove max_batch_size after mamba cache manager is supported
1607-
# ToDo: check 47b and 56b model
1608+
# Once removed max_batch_size, the test will OOM
16081609
kv_cache_config = KvCacheConfig(enable_block_reuse=False)
16091610
with LLM(self.MODEL_PATH,
16101611
kv_cache_config=kv_cache_config,
1611-
max_batch_size=128) as llm:
1612+
max_batch_size=128,
1613+
cuda_graph_config=CudaGraphConfig()
1614+
if cuda_graph else None) as llm:
16121615
task = MMLU(self.MODEL_NAME)
16131616
task.evaluate(llm)
16141617
task = GSM8K(self.MODEL_NAME)
16151618
task.evaluate(llm)
16161619

16171620
@skip_pre_ada
1618-
def test_reasoning_fp8_prequantized(self):
1621+
@parametrize_with_ids("cuda_graph", [False, True])
1622+
def test_reasoning_fp8_prequantized(self, cuda_graph):
16191623
kv_cache_config = KvCacheConfig(enable_block_reuse=False)
16201624
with LLM(f"{llm_models_root()}/Nemotron-H-8B-Reasoning-128K-FP8",
16211625
kv_cache_config=kv_cache_config,
1622-
max_batch_size=256) as llm:
1626+
max_batch_size=256,
1627+
cuda_graph_config=CudaGraphConfig()
1628+
if cuda_graph else None) as llm:
16231629
assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
16241630
task = MMLU(self.MODEL_NAME)
16251631
task.evaluate(llm)
16261632
task = GSM8K(self.MODEL_NAME)
16271633
task.evaluate(llm)
16281634

16291635

1636+
@pytest.mark.skip_less_device(8)
1637+
@pytest.mark.skip_less_device_memory(80000)
1638+
class TestNemotronH_47B_Base(LlmapiAccuracyTestHarness):
1639+
MODEL_NAME = "nvidia/Nemotron-H-47B-Base-8K"
1640+
MODEL_PATH = f"{llm_models_root()}/Nemotron-H-47B-Base-8K"
1641+
1642+
@parametrize_with_ids("cuda_graph", [False, True])
1643+
@pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4),
1644+
(8, 1, 8)],
1645+
ids=["tp8", "tp8ep4", "tp8ep8"])
1646+
def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size):
1647+
kv_cache_config = KvCacheConfig(enable_block_reuse=False,
1648+
free_gpu_memory_fraction=0.6)
1649+
with LLM(self.MODEL_PATH,
1650+
tensor_parallel_size=tp_size,
1651+
pipeline_parallel_size=pp_size,
1652+
moe_expert_parallel_size=ep_size,
1653+
kv_cache_config=kv_cache_config,
1654+
max_batch_size=256,
1655+
cuda_graph_config=CudaGraphConfig()
1656+
if cuda_graph else None) as llm:
1657+
task = MMLU(self.MODEL_NAME)
1658+
task.evaluate(llm)
1659+
task = GSM8K(self.MODEL_NAME)
1660+
task.evaluate(llm)
1661+
1662+
@skip_pre_ada
1663+
@parametrize_with_ids("cuda_graph", [False, True])
1664+
@pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4),
1665+
(8, 1, 8)],
1666+
ids=["tp8", "tp8ep4", "tp8ep8"])
1667+
def test_reasoning_fp8_prequantized(self, cuda_graph, tp_size, pp_size,
1668+
ep_size):
1669+
kv_cache_config = KvCacheConfig(enable_block_reuse=False,
1670+
free_gpu_memory_fraction=0.6)
1671+
with LLM(f"{llm_models_root()}/Nemotron-H-47B-Reasoning-128K-FP8",
1672+
kv_cache_config=kv_cache_config,
1673+
tensor_parallel_size=tp_size,
1674+
pipeline_parallel_size=pp_size,
1675+
moe_expert_parallel_size=ep_size,
1676+
max_batch_size=256,
1677+
cuda_graph_config=CudaGraphConfig()
1678+
if cuda_graph else None) as llm:
1679+
assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
1680+
task = MMLU(self.MODEL_NAME)
1681+
task.evaluate(llm)
1682+
task = GSM8K(self.MODEL_NAME)
1683+
task.evaluate(llm)
1684+
1685+
1686+
@pytest.mark.skip_less_device(8)
1687+
@pytest.mark.skip_less_device_memory(80000)
1688+
class TestNemotronH_56B_Base(LlmapiAccuracyTestHarness):
1689+
MODEL_NAME = "nvidia/Nemotron-H-56B-Base-8K"
1690+
MODEL_PATH = f"{llm_models_root()}/Nemotron-H-56B-Base-8K"
1691+
1692+
@parametrize_with_ids("cuda_graph", [False, True])
1693+
@pytest.mark.parametrize("tp_size,pp_size,ep_size", [(8, 1, 1), (8, 1, 4),
1694+
(8, 1, 8)],
1695+
ids=["tp8", "tp8ep4", "tp8ep8"])
1696+
def test_auto_dtype(self, cuda_graph, tp_size, pp_size, ep_size):
1697+
kv_cache_config = KvCacheConfig(enable_block_reuse=False,
1698+
free_gpu_memory_fraction=0.6)
1699+
with LLM(self.MODEL_PATH,
1700+
tensor_parallel_size=tp_size,
1701+
pipeline_parallel_size=pp_size,
1702+
moe_expert_parallel_size=ep_size,
1703+
kv_cache_config=kv_cache_config,
1704+
max_batch_size=256,
1705+
cuda_graph_config=CudaGraphConfig()
1706+
if cuda_graph else None) as llm:
1707+
task = MMLU(self.MODEL_NAME)
1708+
task.evaluate(llm)
1709+
task = GSM8K(self.MODEL_NAME)
1710+
task.evaluate(llm)
1711+
1712+
16301713
class TestQwen2_7BInstruct(LlmapiAccuracyTestHarness):
16311714
MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
16321715
MODEL_PATH = f"{llm_models_root()}/Qwen2-7B-Instruct"

‎tests/integration/test_lists/qa/benchmark_test_list.txt

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,12 @@ accuracy/test_cli_flow.py::TestNemotronUltra::test_fp8_prequantized[tp8-cuda_gra
1212
accuracy/test_llm_api_pytorch.py::TestLlama3_3NemotronSuper49Bv1::test_fp8_prequantized_tp2
1313
accuracy/test_llm_api_pytorch.py::TestLlama3_1NemotronNano8Bv1::test_auto_dtype
1414
accuracy/test_llm_api_pytorch.py::TestLlama3_1NemotronNano8Bv1::test_fp8_prequantized
15-
accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype
16-
accuracy/test_llm_api_pytorch.py::TestNemotronH::test_reasoning_fp8_prequantized
15+
accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype[cuda_graph=True]
16+
accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype[cuda_graph=False]
17+
accuracy/test_llm_api_pytorch.py::TestNemotronH::test_reasoning_fp8_prequantized[cuda_graph=True]
18+
accuracy/test_llm_api_pytorch.py::TestNemotronH_47B_Base::test_auto_dtype[tp8ep4-cuda_graph=True]
19+
accuracy/test_llm_api_pytorch.py::TestNemotronH_47B_Base::test_reasoning_fp8_prequantized[tp8ep8-cuda_graph=True]
20+
accuracy/test_llm_api_pytorch.py::TestNemotronH_56B_Base::test_auto_dtype[tp8-cuda_graph=True]
1721
accuracy/test_llm_api_pytorch.py::TestNemotronUltra::test_auto_dtype[tp8ep4-cuda_graph=True]
1822
accuracy/test_llm_api_pytorch.py::TestNemotronUltra::test_fp8_prequantized[tp8ep4-cuda_graph=True]
1923
accuracy/test_llm_api_pytorch.py::TestNemotronUltra::test_fp8_prequantized[tp8-cuda_graph=True]

0 commit comments

Comments
 (0)