Skip to content

Commit f36e859

Browse files
nihuinihui
committed
cc
1 parent 816da8f commit f36e859

2 files changed

Lines changed: 93 additions & 301 deletions

File tree

‎tools/quantize/ncnnllm2int.cpp‎

Lines changed: 17 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -206,10 +206,7 @@ static bool read_llm_scale_table(const char* filepath, std::map<std::string, LLM
206206
{
207207
scale.qweight = resolve_qweight_path(filepath, v);
208208
}
209-
else if (strcmp(k, "method") == 0)
210-
{
211-
}
212-
else
209+
else if (strcmp(k, "method") != 0)
213210
{
214211
fprintf(stderr, "%s unsupported metadata %s\n", filepath, k);
215212
fclose(fp);
@@ -418,44 +415,15 @@ static int read_qweight_file(const char* key, const LLMWeightScale& scale, int K
418415

419416
static int llm_table_row_to_qweight(const char* key, const LLMWeightScale& scale, int K, int N, int& weight_bits, int& block_size, ncnn::Mat& weight_data_quantize_scales, ncnn::Mat& weight_data_quantized)
420417
{
421-
if (scale.bits == 0 || scale.block_size == 0 || scale.qweight.empty())
418+
if (scale.qweight.empty())
422419
{
423420
fprintf(stderr, "%s missing mandatory metadata\n", key);
424421
return -1;
425422
}
426423

427-
weight_bits = scale.bits;
428-
block_size = scale.block_size;
429-
430-
if (llm_weight_block_quantize_term(weight_bits, block_size) == 0)
431-
{
432-
fprintf(stderr, "%s unsupported bits=%d block=%d\n", key, weight_bits, block_size);
433-
return -1;
434-
}
435-
436-
const int block_count = (K + block_size - 1) / block_size;
437-
const size_t weight_scale_count = (size_t)N * block_count;
438-
439-
if ((size_t)scale.scales.w != weight_scale_count)
440-
{
441-
fprintf(stderr, "%s coefficient count mismatch expected=%zu got=%d\n", key, weight_scale_count, scale.scales.w);
442-
return -1;
443-
}
444-
445-
weight_data_quantize_scales.create(block_count, N);
446-
if (weight_data_quantize_scales.empty())
447-
return -100;
448-
449-
memcpy(weight_data_quantize_scales.data, scale.scales.data, weight_scale_count * sizeof(float));
450-
const float* scale_ptr = weight_data_quantize_scales;
451-
for (size_t i = 0; i < weight_scale_count; i++)
452-
{
453-
if (!(scale_ptr[i] > 0.f))
454-
{
455-
fprintf(stderr, "%s invalid weight scale index=%zu coeff=%f\n", key, i, scale_ptr[i]);
456-
return -1;
457-
}
458-
}
424+
int ret = llm_table_row_to_scales(key, scale, K, N, weight_bits, block_size, weight_data_quantize_scales);
425+
if (ret != 0)
426+
return ret;
459427

460428
return read_qweight_file(key, scale, K, N, weight_bits, weight_data_quantized);
461429
}
@@ -509,7 +477,7 @@ int NetQuantize::quantize_gemm(int block_size, int weight_bits, int method)
509477
continue;
510478
}
511479

512-
fprintf(stderr, "quantize_gemm bits=%d block_size=%d term=%d %s\n", weight_bits, block_size, quantize_term, gemm_name(gemm));
480+
fprintf(stderr, "quantize_gemm %s\n", gemm_name(gemm));
513481

514482
ncnn::Mat B_data_quantized;
515483
ncnn::Mat B_data_quantize_scales;
@@ -599,7 +567,7 @@ int NetQuantize::quantize_gemm_from_table(std::map<std::string, LLMWeightScale>&
599567
}
600568

601569
const int quantize_term = llm_weight_block_quantize_term(weight_bits, block_size, has_input_scale);
602-
fprintf(stderr, "quantize_gemm table bits=%d block_size=%d term=%d %s\n", weight_bits, block_size, quantize_term, gemm_name(gemm));
570+
fprintf(stderr, "quantize_gemm %s\n", gemm_name(gemm));
603571

604572
if (!qweight_row)
605573
{
@@ -661,7 +629,7 @@ int NetQuantize::quantize_multiheadattention(int block_size, int weight_bits, in
661629
continue;
662630
}
663631

664-
fprintf(stderr, "quantize_multiheadattention bits=%d block_size=%d term=%d %s\n", weight_bits, block_size, quantize_term, multiheadattention_name(mha));
632+
fprintf(stderr, "quantize_multiheadattention %s\n", multiheadattention_name(mha));
665633

666634
const int qdim = mha->weight_data_size / mha->embed_dim;
667635

@@ -767,13 +735,10 @@ int NetQuantize::quantize_multiheadattention_from_table(std::map<std::string, LL
767735
input_scale_present_count++;
768736
}
769737

770-
if (input_scale_present_count != 0)
738+
if (input_scale_present_count != 0 && input_scale_present_count != 4)
771739
{
772-
if (input_scale_present_count != 4)
773-
{
774-
fprintf(stderr, "MultiHeadAttention %s requires all input scale table rows %s %s %s %s\n", multiheadattention_name(mha), input_scale_keys[0], input_scale_keys[1], input_scale_keys[2], input_scale_keys[3]);
775-
return -1;
776-
}
740+
fprintf(stderr, "MultiHeadAttention %s requires all input scale table rows %s %s %s %s\n", multiheadattention_name(mha), input_scale_keys[0], input_scale_keys[1], input_scale_keys[2], input_scale_keys[3]);
741+
return -1;
777742
}
778743

779744
int weight_bits[4];
@@ -811,17 +776,12 @@ int NetQuantize::quantize_multiheadattention_from_table(std::map<std::string, LL
811776

812777
ncnn::Mat input_scales[4];
813778
const bool has_input_scale = input_scale_present_count == 4;
814-
if (has_input_scale)
779+
if (has_input_scale && qweight_rows[0])
815780
{
816-
for (int j = 0; j < 4; j++)
817-
{
818-
if (qweight_rows[j])
819-
{
820-
fprintf(stderr, "MultiHeadAttention %s does not support input_scale with qweight yet\n", multiheadattention_name(mha));
821-
return -1;
822-
}
823-
}
781+
fprintf(stderr, "MultiHeadAttention %s does not support input_scale with qweight yet\n", multiheadattention_name(mha));
782+
return -1;
824783
}
784+
825785
if (has_input_scale)
826786
{
827787
for (int j = 0; j < 4; j++)
@@ -833,7 +793,7 @@ int NetQuantize::quantize_multiheadattention_from_table(std::map<std::string, LL
833793
}
834794

835795
const int quantize_term = llm_weight_block_quantize_term(weight_bits[0], block_size[0], has_input_scale);
836-
fprintf(stderr, "quantize_multiheadattention table bits=%d block_size=%d term=%d %s\n", weight_bits[0], block_size[0], quantize_term, multiheadattention_name(mha));
796+
fprintf(stderr, "quantize_multiheadattention %s\n", multiheadattention_name(mha));
837797

838798
const ncnn::Mat q_weight_data = mha->q_weight_data.reshape(qdim, mha->embed_dim);
839799
const ncnn::Mat k_weight_data = mha->k_weight_data.reshape(mha->kdim, mha->embed_dim);
@@ -998,7 +958,7 @@ int main(int argc, char** argv)
998958
}
999959

1000960
NetQuantize quantizer;
1001-
quantizer.storage_type = 1; // keep existing prototype behavior for unrelated fp32 weights
961+
quantizer.storage_type = 1; // use fp16 where weight quant not applied
1002962

1003963
if (quantizer.load_param(inparam) != 0)
1004964
return -1;

0 commit comments

Comments
 (0)