Skip to content

Commit d59876e

Browse files
committed
Using the same weight_pointers key, added a TODO for a missing torch tensor conversion for is_dora
Signed-off-by: Amit Zuker <[email protected]>
1 parent eb4c815 commit d59876e

File tree

1 file changed

+4
-2
lines changed

1 file changed

+4
-2
lines changed

tensorrt_llm/_torch/pyexecutor/model_engine.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2052,7 +2052,7 @@ def _get_lora_params_from_requests(self,
20522052
module_id)] = {
20532053
'adapter_size': [module.adapter_size],
20542054
'is_dora': [scaling_vec_pointer == 0],
2055-
'weights_pointer': [
2055+
'weight_pointers': [
20562056
module.weights_in_pointer,
20572057
module.weights_out_pointer,
20582058
scaling_vec_pointer
@@ -2087,11 +2087,13 @@ def _get_lora_params_from_requests(self,
20872087
'is_dora'] += current_tmp_lora_params['is_dora']
20882088
current_lora_params[
20892089
'weight_pointers'] += current_tmp_lora_params[
2090-
'weights_pointer']
2090+
'weight_pointers']
20912091

20922092
for layer_id in lora_params:
20932093
for module_id in lora_params[layer_id]:
20942094
current_lora_params = lora_params[layer_id][module_id]
2095+
# TODO: When lora_grouped_gemm supports DoRA: convert 'is_dora' to a bool tensor.
2096+
# Until it's supported, that would just slow down this function, so better not to do it.
20952097
current_lora_params['adapter_size'] = torch.IntTensor(
20962098
current_lora_params['adapter_size'])
20972099
current_lora_params['weight_pointers'] = torch.LongTensor(

0 commit comments

Comments
 (0)