Skip to content

Commit dc5139d

Browse files
nihuinihui
committed
w
1 parent f36e859 commit dc5139d

4 files changed

Lines changed: 33 additions & 13 deletions

File tree

‎src/layer/gemm.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,11 @@ class Gemm : public Layer
6868
Mat A_data_int8_scales;
6969
float B_data_int8_scale;
7070
#endif
71+
72+
#if NCNN_WEIGHT_QUANT
7173
Mat B_data_quantize_scales;
7274
Mat B_data_input_scales;
75+
#endif
7376
};
7477

7578
} // namespace ncnn

‎src/layer/multiheadattention.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ class MultiHeadAttention : public Layer
6262
Mat v_weight_data_int8_scales;
6363
float out_weight_data_int8_scale;
6464
#endif
65+
66+
#if NCNN_WEIGHT_QUANT
6567
Mat q_weight_data_quantize_scales;
6668
Mat k_weight_data_quantize_scales;
6769
Mat v_weight_data_quantize_scales;
@@ -70,6 +72,7 @@ class MultiHeadAttention : public Layer
7072
Mat k_weight_data_input_scales;
7173
Mat v_weight_data_input_scales;
7274
Mat out_weight_data_input_scales;
75+
#endif
7376
};
7477

7578
} // namespace ncnn

‎tools/modelwriter.h‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
#include "layer/yolodetectionoutput.h"
110110
#include "layer/yolov3detectionoutput.h"
111111

112+
#if NCNN_WEIGHT_QUANT
112113
static bool modelwriter_is_weight_block_quantize(int quantize_term)
113114
{
114115
const int weight_bits = quantize_term / 100;
@@ -122,6 +123,7 @@ static bool modelwriter_weight_block_quantize_has_input_scale(int quantize_term)
122123
{
123124
return quantize_term % 100 / 10 == 1;
124125
}
126+
#endif // NCNN_WEIGHT_QUANT
125127

126128
// for gen_random_weight
127129
#include "../tests/prng.h"
@@ -1895,8 +1897,11 @@ int ModelWriter::save(const char* parampath, const char* binpath)
18951897
fprintf_param_value(" 21=%d", constant_TILE_N)
18961898
fprintf_param_value(" 22=%d", constant_TILE_K)
18971899

1898-
const bool weight_block_quantize
1899-
= modelwriter_is_weight_block_quantize(op->quantize_term);
1900+
#if NCNN_WEIGHT_QUANT
1901+
const bool weight_block_quantize = modelwriter_is_weight_block_quantize(op->quantize_term);
1902+
#else
1903+
const bool weight_block_quantize = false;
1904+
#endif
19001905

19011906
if (op->constantA == 1)
19021907
{
@@ -1918,6 +1923,7 @@ int ModelWriter::save(const char* parampath, const char* binpath)
19181923
fwrite_weight_tag_data(op->C_data, bp);
19191924
}
19201925

1926+
#if NCNN_WEIGHT_QUANT
19211927
if (weight_block_quantize)
19221928
{
19231929
if (op->constantB == 1)
@@ -1929,6 +1935,7 @@ int ModelWriter::save(const char* parampath, const char* binpath)
19291935
}
19301936
}
19311937
}
1938+
#endif // NCNN_WEIGHT_QUANT
19321939
#if NCNN_INT8
19331940
// write int8_scale data
19341941
if (op->quantize_term && !weight_block_quantize)
@@ -2198,8 +2205,11 @@ int ModelWriter::save(const char* parampath, const char* binpath)
21982205
fprintf_param_value(" 7=%d", kv_cache)
21992206
fprintf_param_value(" 18=%d", quantize_term)
22002207

2201-
const bool weight_block_quantize
2202-
= modelwriter_is_weight_block_quantize(op->quantize_term);
2208+
#if NCNN_WEIGHT_QUANT
2209+
const bool weight_block_quantize = modelwriter_is_weight_block_quantize(op->quantize_term);
2210+
#else
2211+
const bool weight_block_quantize = false;
2212+
#endif
22032213

22042214
if (weight_block_quantize)
22052215
{
@@ -2238,6 +2248,7 @@ int ModelWriter::save(const char* parampath, const char* binpath)
22382248
}
22392249
fwrite_weight_data(op->out_bias_data, bp);
22402250

2251+
#if NCNN_WEIGHT_QUANT
22412252
if (weight_block_quantize)
22422253
{
22432254
fwrite_weight_data(op->q_weight_data_quantize_scales, bp, false);
@@ -2252,9 +2263,10 @@ int ModelWriter::save(const char* parampath, const char* binpath)
22522263
fwrite_weight_data(op->out_weight_data_input_scales, bp, false);
22532264
}
22542265
}
2266+
#endif // NCNN_WEIGHT_QUANT
22552267
#if NCNN_INT8
22562268
// write int8_scale data
2257-
else if (op->quantize_term)
2269+
if (op->quantize_term && !weight_block_quantize)
22582270
{
22592271
fwrite_weight_data(op->q_weight_data_int8_scales, bp, 90, 100);
22602272
fwrite_weight_data(op->k_weight_data_int8_scales, bp, 90, 100);

‎tools/quantize/CMakeLists.txt‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11

2-
add_executable(ncnnllm2int ncnnllm2int.cpp)
3-
target_link_libraries(ncnnllm2int PRIVATE ncnn)
2+
if(NCNN_WEIGHT_QUANT)
3+
add_executable(ncnnllm2int ncnnllm2int.cpp)
4+
target_link_libraries(ncnnllm2int PRIVATE ncnn)
45

5-
add_executable(ncnnllm2table ncnnllm2table.cpp)
6-
target_link_libraries(ncnnllm2table PRIVATE ncnn)
6+
add_executable(ncnnllm2table ncnnllm2table.cpp)
7+
target_link_libraries(ncnnllm2table PRIVATE ncnn)
78

8-
set_property(TARGET ncnnllm2int PROPERTY FOLDER "tools/optimization")
9-
set_property(TARGET ncnnllm2table PROPERTY FOLDER "tools/optimization")
10-
ncnn_install_tool(ncnnllm2int)
11-
ncnn_install_tool(ncnnllm2table)
9+
set_property(TARGET ncnnllm2int PROPERTY FOLDER "tools/optimization")
10+
set_property(TARGET ncnnllm2table PROPERTY FOLDER "tools/optimization")
11+
ncnn_install_tool(ncnnllm2int)
12+
ncnn_install_tool(ncnnllm2table)
13+
endif()
1214

1315
if(NCNN_INT8)
1416
if(NCNN_PIXEL)

0 commit comments

Comments
 (0)