Set float32 MLP output dtype for Qwen3

turboderp · turboderp · commit 4724b86d2f06 · 2025-04-30T17:55:16.000+02:00
diff --git a/exllamav3/models/qwen3.py b/exllamav3/models/qwen3.py
@@ -115,6 +115,8 @@ def __init__(
                     key_gate = "gate_proj",
                     key_down = "down_proj",
                     qmap = "block.mlp",
+                    interm_dtype = torch.half,
+                    out_dtype = torch.float,
                 ),
             )
             for idx in range(config.num_hidden_layers)

Original file line number	Diff line number	Diff line change
`@@ -115,6 +115,8 @@ def __init__(`
`115`	`115`	`key_gate = "gate_proj",`
`116`	`116`	`key_down = "down_proj",`
`117`	`117`	`qmap = "block.mlp",`
	`118`	`+ interm_dtype = torch.half,`
	`119`	`+ out_dtype = torch.float,`
`118`	`120`	`),`
`119`	`121`	`)`
`120`	`122`	`for idx in range(config.num_hidden_layers)`