Skip to content

Commit 6e58ed3

Browse files
prabhuprabhu
authored
cdx1-mini (#2148)
Signed-off-by: Prabhu Subramanian <[email protected]>
1 parent 216af8f commit 6e58ed3

File tree

16 files changed

+5001
-231
lines changed

16 files changed

+5001
-231
lines changed

‎contrib/fine-tuning/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
dataset/
22
adapters/
3+
adapters-mini/
34
out.txt
45
unsloth/
56
CycloneDX/
7+
Qwen/

‎contrib/fine-tuning/Modelfile-mini‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
FROM ./cdx1-mini-4B-q8_0.gguf
2+
3+
PARAMETER num_ctx 16000
4+
PARAMETER temperature 0.7
5+
6+
SYSTEM """You are a helpful assistant to the user."""
7+
8+
LICENSE """
9+
apache-2.0
10+
"""

‎contrib/fine-tuning/convert-gguf.sh‎

Lines changed: 76 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,21 @@ set -e
1717
# cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
1818

1919
export TOKENIZERS_PARALLELISM=false
20+
LLAMA_CPP_PATH=/Users/appthreat/work/llama.cpp
21+
cd $LLAMA_CPP_PATH
22+
source .venv/bin/activate
23+
CDXGEN_FT_PATH=/Users/appthreat/work/cdxgen/contrib/fine-tuning
2024

2125
TUNING_TOOL=mlx
2226
FORMAT=GGUF
2327
HF_ORG=CycloneDX
2428
TOOL_BASE_MODEL=${1:-cdx1}
29+
MODEL_FILE_PATH=${CDXGEN_FT_PATH}/Modelfile
2530
case "$TOOL_BASE_MODEL" in
31+
cdx1-mini)
32+
PARAM_SIZE="4B"
33+
MODEL_FILE_PATH=${CDXGEN_FT_PATH}/Modelfile-mini
34+
;;
2635
cdx1-pro)
2736
PARAM_SIZE="30B"
2837
;;
@@ -33,10 +42,6 @@ case "$TOOL_BASE_MODEL" in
3342
PARAM_SIZE="gguf"
3443
;;
3544
esac
36-
LLAMA_CPP_PATH=/Users/appthreat/work/llama.cpp
37-
cd $LLAMA_CPP_PATH
38-
source .venv/bin/activate
39-
CDXGEN_FT_PATH=/Users/appthreat/work/cdxgen/contrib/fine-tuning
4045

4146
GGUF_MODEL_Q8_0_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q8_0-${FORMAT}
4247
GGUF_MODEL_Q8_0_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q8_0-${FORMAT}
@@ -45,44 +50,55 @@ FUSED_MODEL=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${TUNING_TOOL}
4550
rm -rf ${GGUF_MODEL_Q8_0_PATH}
4651
mkdir -p ${GGUF_MODEL_Q8_0_PATH}
4752
python convert_hf_to_gguf.py --outtype q8_0 --outfile ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q8_0-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-q8_0.gguf --model-name ${GGUF_MODEL_Q8_0_NAME} ${FUSED_MODEL}
48-
cp ${CDXGEN_FT_PATH}/Modelfile ${GGUF_MODEL_Q8_0_PATH}/
53+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_Q8_0_PATH}/Modelfile
4954
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q8_0_PATH}/
5055

5156
GGUF_MODEL_BF16_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}
5257
GGUF_MODEL_BF16_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}
5358
rm -rf ${GGUF_MODEL_BF16_PATH}
5459
mkdir -p ${GGUF_MODEL_BF16_PATH}
5560
python convert_hf_to_gguf.py --outtype bf16 --outfile ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf --model-name ${GGUF_MODEL_BF16_NAME} ${FUSED_MODEL}
56-
cp ${CDXGEN_FT_PATH}/Modelfile ${GGUF_MODEL_BF16_PATH}/
61+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_BF16_PATH}/Modelfile
5762
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-bf16.gguf|g' ${GGUF_MODEL_BF16_PATH}/Modelfile
5863
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_BF16_PATH}/
5964

60-
GGUF_MODEL_Q4_K_M_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M-${FORMAT}
61-
GGUF_MODEL_Q4_K_M_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M-${FORMAT}
62-
rm -rf ${GGUF_MODEL_Q4_K_M_PATH}
63-
mkdir -p ${GGUF_MODEL_Q4_K_M_PATH}
64-
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_Q4_K_M_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M.gguf Q4_K_M
65-
cp ${CDXGEN_FT_PATH}/Modelfile ${GGUF_MODEL_Q4_K_M_PATH}/
66-
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-Q4_K_M.gguf|g' ${GGUF_MODEL_Q4_K_M_PATH}/Modelfile
67-
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q4_K_M_PATH}/
68-
69-
GGUF_MODEL_IQ4_NL_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL-${FORMAT}
70-
GGUF_MODEL_IQ4_NL_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL-${FORMAT}
71-
rm -rf ${GGUF_MODEL_IQ4_NL_PATH}
72-
mkdir -p ${GGUF_MODEL_IQ4_NL_PATH}
73-
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_IQ4_NL_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL.gguf IQ4_NL
74-
cp ${CDXGEN_FT_PATH}/Modelfile ${GGUF_MODEL_IQ4_NL_PATH}/
75-
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-IQ4_NL.gguf|g' ${GGUF_MODEL_IQ4_NL_PATH}/Modelfile
76-
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_IQ4_NL_PATH}/
77-
78-
GGUF_MODEL_Q2_K_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K-${FORMAT}
79-
GGUF_MODEL_Q2_K_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K-${FORMAT}
80-
rm -rf ${GGUF_MODEL_Q2_K_PATH}
81-
mkdir -p ${GGUF_MODEL_Q2_K_PATH}
82-
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_Q2_K_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K.gguf Q2_K
83-
cp ${CDXGEN_FT_PATH}/Modelfile ${GGUF_MODEL_Q2_K_PATH}/
84-
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-Q2_K.gguf|g' ${GGUF_MODEL_Q2_K_PATH}/Modelfile
85-
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q2_K_PATH}/
65+
if [ "$TOOL_BASE_MODEL" == "cdx1-mini" ]; then
66+
GGUF_MODEL_Q6_K_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q6_K-${FORMAT}
67+
GGUF_MODEL_Q6_K_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q6_K-${FORMAT}
68+
rm -rf ${GGUF_MODEL_Q6_K_PATH}
69+
mkdir -p ${GGUF_MODEL_Q6_K_PATH}
70+
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_Q6_K_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q6_K.gguf Q6_K
71+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_Q6_K_PATH}/Modelfile
72+
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-Q6_K.gguf|g' ${GGUF_MODEL_Q6_K_PATH}/Modelfile
73+
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q6_K_PATH}/
74+
else
75+
GGUF_MODEL_Q4_K_M_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M-${FORMAT}
76+
GGUF_MODEL_Q4_K_M_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M-${FORMAT}
77+
rm -rf ${GGUF_MODEL_Q4_K_M_PATH}
78+
mkdir -p ${GGUF_MODEL_Q4_K_M_PATH}
79+
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_Q4_K_M_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q4_K_M.gguf Q4_K_M
80+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_Q4_K_M_PATH}/Modelfile
81+
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-Q4_K_M.gguf|g' ${GGUF_MODEL_Q4_K_M_PATH}/Modelfile
82+
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q4_K_M_PATH}/
83+
84+
GGUF_MODEL_IQ4_NL_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL-${FORMAT}
85+
GGUF_MODEL_IQ4_NL_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL-${FORMAT}
86+
rm -rf ${GGUF_MODEL_IQ4_NL_PATH}
87+
mkdir -p ${GGUF_MODEL_IQ4_NL_PATH}
88+
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_IQ4_NL_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-IQ4_NL.gguf IQ4_NL
89+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_IQ4_NL_PATH}/Modelfile
90+
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-IQ4_NL.gguf|g' ${GGUF_MODEL_IQ4_NL_PATH}/Modelfile
91+
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_IQ4_NL_PATH}/
92+
93+
GGUF_MODEL_Q2_K_NAME=${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K-${FORMAT}
94+
GGUF_MODEL_Q2_K_PATH=${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K-${FORMAT}
95+
rm -rf ${GGUF_MODEL_Q2_K_PATH}
96+
mkdir -p ${GGUF_MODEL_Q2_K_PATH}
97+
llama-quantize ${CDXGEN_FT_PATH}/${HF_ORG}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-BF16-${FORMAT}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-bf16.gguf ${GGUF_MODEL_Q2_K_PATH}/${TOOL_BASE_MODEL}-${PARAM_SIZE}-Q2_K.gguf Q2_K
98+
cp ${MODEL_FILE_PATH} ${GGUF_MODEL_Q2_K_PATH}/Modelfile
99+
sed -i '' 's|./cdx1-${PARAM_SIZE}-q8_0.gguf|./cdx1-${PARAM_SIZE}-Q2_K.gguf|g' ${GGUF_MODEL_Q2_K_PATH}/Modelfile
100+
cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q2_K_PATH}/
101+
fi
86102

87103
### Testing with ollama
88104
# cd ${GGUF_MODEL_Q8_0_PATH}
@@ -93,30 +109,41 @@ cp ${FUSED_MODEL}/*.json ${FUSED_MODEL}/merges.txt ${GGUF_MODEL_Q2_K_PATH}/
93109
export HF_HUB_ENABLE_HF_TRANSFER=0
94110
hf auth whoami
95111
hf upload --quiet --repo-type model ${GGUF_MODEL_Q8_0_NAME} ${GGUF_MODEL_Q8_0_PATH} .
96-
hf upload --quiet --repo-type model ${GGUF_MODEL_Q4_K_M_NAME} ${GGUF_MODEL_Q4_K_M_PATH} .
97-
hf upload --quiet --repo-type model ${GGUF_MODEL_IQ4_NL_NAME} ${GGUF_MODEL_IQ4_NL_PATH} .
98-
hf upload --quiet --repo-type model ${GGUF_MODEL_Q2_K_NAME} ${GGUF_MODEL_Q2_K_PATH} .
112+
if [ "$TOOL_BASE_MODEL" == "cdx1-mini" ]; then
113+
hf upload --quiet --repo-type model ${GGUF_MODEL_Q6_K_NAME} ${GGUF_MODEL_Q6_K_PATH} .
114+
else
115+
hf upload --quiet --repo-type model ${GGUF_MODEL_Q4_K_M_NAME} ${GGUF_MODEL_Q4_K_M_PATH} .
116+
hf upload --quiet --repo-type model ${GGUF_MODEL_IQ4_NL_NAME} ${GGUF_MODEL_IQ4_NL_PATH} .
117+
hf upload --quiet --repo-type model ${GGUF_MODEL_Q2_K_NAME} ${GGUF_MODEL_Q2_K_PATH} .
118+
fi
99119
hf upload --quiet --repo-type model ${GGUF_MODEL_BF16_NAME} ${GGUF_MODEL_BF16_PATH} .
100120

101121
ollama pull hf.co/${GGUF_MODEL_Q8_0_NAME}
102122
ollama cp hf.co/${GGUF_MODEL_Q8_0_NAME} ${GGUF_MODEL_Q8_0_NAME}
103123
ollama push ${GGUF_MODEL_Q8_0_NAME}
104124
ollama rm hf.co/${GGUF_MODEL_Q8_0_NAME}
105125

106-
ollama pull hf.co/${GGUF_MODEL_Q4_K_M_NAME}
107-
ollama cp hf.co/${GGUF_MODEL_Q4_K_M_NAME} ${GGUF_MODEL_Q4_K_M_NAME}
108-
ollama push ${GGUF_MODEL_Q4_K_M_NAME}
109-
ollama rm hf.co/${GGUF_MODEL_Q4_K_M_NAME}
110-
111-
ollama pull hf.co/${GGUF_MODEL_IQ4_NL_NAME}
112-
ollama cp hf.co/${GGUF_MODEL_IQ4_NL_NAME} ${GGUF_MODEL_IQ4_NL_NAME}
113-
ollama push ${GGUF_MODEL_IQ4_NL_NAME}
114-
ollama rm hf.co/${GGUF_MODEL_IQ4_NL_NAME}
115-
116-
ollama pull hf.co/${GGUF_MODEL_Q2_K_NAME}
117-
ollama cp hf.co/${GGUF_MODEL_Q2_K_NAME} ${GGUF_MODEL_Q2_K_NAME}
118-
ollama push ${GGUF_MODEL_Q2_K_NAME}
119-
ollama rm hf.co/${GGUF_MODEL_Q2_K_NAME}
126+
if [ "$TOOL_BASE_MODEL" == "cdx1-mini" ]; then
127+
ollama pull hf.co/${GGUF_MODEL_Q6_K_NAME}
128+
ollama cp hf.co/${GGUF_MODEL_Q6_K_NAME} ${GGUF_MODEL_Q6_K_NAME}
129+
ollama push ${GGUF_MODEL_Q6_K_NAME}
130+
ollama rm hf.co/${GGUF_MODEL_Q6_K_NAME}
131+
else
132+
ollama pull hf.co/${GGUF_MODEL_Q4_K_M_NAME}
133+
ollama cp hf.co/${GGUF_MODEL_Q4_K_M_NAME} ${GGUF_MODEL_Q4_K_M_NAME}
134+
ollama push ${GGUF_MODEL_Q4_K_M_NAME}
135+
ollama rm hf.co/${GGUF_MODEL_Q4_K_M_NAME}
136+
137+
ollama pull hf.co/${GGUF_MODEL_IQ4_NL_NAME}
138+
ollama cp hf.co/${GGUF_MODEL_IQ4_NL_NAME} ${GGUF_MODEL_IQ4_NL_NAME}
139+
ollama push ${GGUF_MODEL_IQ4_NL_NAME}
140+
ollama rm hf.co/${GGUF_MODEL_IQ4_NL_NAME}
141+
142+
ollama pull hf.co/${GGUF_MODEL_Q2_K_NAME}
143+
ollama cp hf.co/${GGUF_MODEL_Q2_K_NAME} ${GGUF_MODEL_Q2_K_NAME}
144+
ollama push ${GGUF_MODEL_Q2_K_NAME}
145+
ollama rm hf.co/${GGUF_MODEL_Q2_K_NAME}
146+
fi
120147

121148
ollama pull hf.co/${GGUF_MODEL_BF16_NAME}
122149
ollama cp hf.co/${GGUF_MODEL_BF16_NAME} ${GGUF_MODEL_BF16_NAME}

‎contrib/fine-tuning/fine-tune-mlx.sh‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,26 @@
33
set -e
44
TUNING_TOOL=mlx
55
TOOL_BASE_MODEL=${1:-cdx1}
6+
MAX_SEQ=262144
7+
MAX_TOKENS=16384
8+
ITERS=1500
9+
NUM_LAYERS=48
610
case "$TOOL_BASE_MODEL" in
11+
cdx1-mini)
12+
NUM_LAYERS=36
13+
BASE_MODEL="unsloth/Qwen3-4B-Instruct-2507"
14+
;;
715
cdx1-pro)
16+
ITERS=2500
817
BASE_MODEL="unsloth/Qwen3-Coder-30B-A3B-Instruct"
918
;;
1019
*)
20+
ITERS=2000
1121
BASE_MODEL="unsloth/Qwen2.5-Coder-14B-Instruct"
1222
;;
1323
esac
1424
BASE_MODEL_MLX=${BASE_MODEL}-${TUNING_TOOL}
1525
HF_ORG=CycloneDX
16-
NUM_LAYERS=16
1726
ADAPTERS_PATH=adapters
1827
DATASET_PATH=dataset
1928

@@ -38,11 +47,19 @@ node validator.js ${DATASET_PATH}
3847

3948
# This step always pulls the latest base model from HF. Need to think about versioning and checksum to prevent model injection attacks
4049
echo "Test base model with the prompt 'Tell me about cdxgen'. Usually yields a low-quality response."
41-
mlx_lm.generate --model ${BASE_MODEL} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
50+
mlx_lm.generate --model ${BASE_MODEL} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
4251

4352
# We use LoRA fine-tuning over DoRA due to better compatibility with vLLM and llama.cpp
44-
echo "Low-Rank Adaptation (LoRA) fine-tuning ${BASE_MODEL} with cdx1 dataset. This might take a while ..."
45-
mlx_lm.lora --model ${BASE_MODEL} --train --data ${DATASET_PATH} --adapter-path ${ADAPTERS_PATH} --mask-prompt --fine-tune-type lora --batch-size 1 --num-layers ${NUM_LAYERS} --iters 2000 --grad-checkpoint --max-seq-length 16000 --learning-rate "3e-5" --optimizer adam
53+
if [ "$TOOL_BASE_MODEL" = "cdx1-mini" ]; then
54+
echo "Full fine-tune with cdx-docs dataset. This might take a while ..."
55+
mlx_lm.lora --model ${BASE_MODEL} --train --data ${DATASET_PATH} --adapter-path ${ADAPTERS_PATH} --mask-prompt --fine-tune-type full --batch-size 2 --num-layers ${NUM_LAYERS} --iters ${ITERS} --grad-checkpoint --max-seq-length ${MAX_SEQ} --learning-rate "1e-5" --optimizer adamw
56+
elif [ "$TOOL_BASE_MODEL" = "cdx1" ]; then
57+
echo "Low-Rank Adaptation (LoRA) fine-tuning ${BASE_MODEL} with cdx-docs dataset. This might take a while ..."
58+
mlx_lm.lora --model ${BASE_MODEL} --train --data ${DATASET_PATH} --adapter-path ${ADAPTERS_PATH} --mask-prompt --fine-tune-type lora --batch-size 1 --num-layers ${NUM_LAYERS} --iters ${ITERS} --grad-checkpoint --max-seq-length ${MAX_SEQ} --learning-rate "1e-4" --optimizer adamw
59+
else
60+
echo "Low-Rank Adaptation (LoRA) fine-tuning ${BASE_MODEL} with cdx-docs dataset. This might take a while ..."
61+
mlx_lm.lora --model ${BASE_MODEL} --train --data ${DATASET_PATH} --adapter-path ${ADAPTERS_PATH} --mask-prompt --fine-tune-type lora --batch-size 1 --num-layers ${NUM_LAYERS} --iters ${ITERS} --grad-checkpoint --max-seq-length ${MAX_SEQ} --learning-rate "1e-4" --optimizer adamw
62+
fi
4663

4764
echo "Fuse model to ${FUSED_MODEL} using the cdx1 adapters"
4865
rm -rf ${FUSED_MODEL}
@@ -51,7 +68,7 @@ rm -rf ${FUSED_MODEL}
5168
mlx_lm.fuse --model ${BASE_MODEL} --adapter-path adapters --save-path ${FUSED_MODEL} --de-quantize
5269

5370
echo "Test fused model with the prompt 'Tell me about cdxgen'. Must yield a better response."
54-
mlx_lm.generate --model ./${FUSED_MODEL} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
71+
mlx_lm.generate --model ./${FUSED_MODEL} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
5572

5673
rm -rf ${BASE_MODEL_MLX}
5774
mlx_lm.convert --hf-path ${BASE_MODEL} --mlx-path ${BASE_MODEL_MLX}
@@ -67,19 +84,25 @@ echo "Create quantized models"
6784
rm -rf ${QUANT_MODEL_8BIT}
6885
mlx_lm.convert --hf-path ${FUSED_MODEL} --mlx-path ${QUANT_MODEL_8BIT} -q --q-bits 8 --dtype bfloat16
6986
echo "Test ${QUANT_MODEL_8BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
70-
mlx_lm.generate --model ./${QUANT_MODEL_8BIT} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
87+
mlx_lm.generate --model ./${QUANT_MODEL_8BIT} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
7188

7289
rm -rf ${QUANT_MODEL_6BIT}
7390
mlx_lm.convert --hf-path ${FUSED_MODEL} --mlx-path ${QUANT_MODEL_6BIT} -q --q-bits 6 --dtype bfloat16
7491
echo "Test ${QUANT_MODEL_6BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
75-
mlx_lm.generate --model ./${QUANT_MODEL_6BIT} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
92+
mlx_lm.generate --model ./${QUANT_MODEL_6BIT} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
7693

77-
rm -rf ${QUANT_MODEL_4BIT}
78-
mlx_lm.convert --hf-path ${FUSED_MODEL} --mlx-path ${QUANT_MODEL_4BIT} -q --q-bits 4 --dtype bfloat16
79-
echo "Test ${QUANT_MODEL_4BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
80-
mlx_lm.generate --model ./${QUANT_MODEL_4BIT} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
94+
# 4-bit for a small model has very poor performance
95+
if [ "$TOOL_BASE_MODEL" != "cdx1-mini" ]; then
96+
rm -rf ${QUANT_MODEL_4BIT}
97+
mlx_lm.convert --hf-path ${FUSED_MODEL} --mlx-path ${QUANT_MODEL_4BIT} -q --q-bits 4 --dtype bfloat16
98+
echo "Test ${QUANT_MODEL_4BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
99+
mlx_lm.generate --model ./${QUANT_MODEL_4BIT} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
100+
fi
81101

82-
#echo "Generating DWQ Quantized model ${DWQ_QUANT_MODEL_4BIT} with the teacher model ${FUSED_MODEL}. This might take several hours ..."
83-
#mlx_lm.dwq --model ${FUSED_MODEL} --quantized-model ${QUANT_MODEL_8BIT} --mlx-path ${DWQ_QUANT_MODEL_4BIT} --learning-rate "2e-5" --batch-size 1 --data-path dataset --grad-checkpoint
84-
#echo "Test ${DWQ_QUANT_MODEL_4BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
85-
#mlx_lm.generate --model ./${DWQ_QUANT_MODEL_4BIT} --prompt "Tell me about cdxgen" --temp 0.05 --max-tokens 32000
102+
#if [ "$TOOL_BASE_MODEL" = "cdx1-mini" ]; then
103+
# rm -rf ${DWQ_QUANT_MODEL_4BIT}
104+
# echo "Generating DWQ Quantized model ${DWQ_QUANT_MODEL_4BIT} with the teacher model ${FUSED_MODEL}. This might take several hours ..."
105+
# mlx_lm.dwq --model ${FUSED_MODEL} --quantized-model ${QUANT_MODEL_8BIT} --mlx-path ${DWQ_QUANT_MODEL_4BIT} --learning-rate "2e-5" --batch-size 1 --data-path dataset --grad-checkpoint
106+
# echo "Test ${DWQ_QUANT_MODEL_4BIT} with the prompt 'Tell me about cdxgen'. Must yield a better response."
107+
# mlx_lm.generate --model ./${DWQ_QUANT_MODEL_4BIT} --prompt "Tell me about cdxgen" --temp 0.7 --max-tokens ${MAX_TOKENS}
108+
#fi

‎contrib/fine-tuning/upload-hf.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ hf upload --quiet --repo-type dataset CycloneDX/cdx-docs ./semantics semantics
2222
echo "Uploading models. Please wait ..."
2323
hf upload --quiet --repo-type model ${QUANT_MODEL_8BIT} ./${QUANT_MODEL_8BIT} .
2424
hf upload --quiet --repo-type model ${QUANT_MODEL_6BIT} ./${QUANT_MODEL_6BIT} .
25-
hf upload --quiet --repo-type model ${QUANT_MODEL_4BIT} ./${QUANT_MODEL_4BIT} .
26-
#hf upload --quiet --repo-type model ${DWQ_QUANT_MODEL_4BIT} ./${DWQ_QUANT_MODEL_4BIT} .
27-
25+
if [ "$TOOL_BASE_MODEL" != "cdx1-mini" ]; then
26+
hf upload --quiet --repo-type model ${QUANT_MODEL_4BIT} ./${QUANT_MODEL_4BIT} .
27+
fi
28+
#if [ "$TOOL_BASE_MODEL" = "cdx1-mini" ]; then
29+
# hf upload --quiet --repo-type model ${DWQ_QUANT_MODEL_4BIT} ./${DWQ_QUANT_MODEL_4BIT} .
30+
#fi
2831
hf upload --quiet --repo-type model ${FUSED_MODEL} ./${FUSED_MODEL} .

‎contrib/xBOMEval/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ xBOMEval is a benchmark useful for evaluating LLMs that are optimised for BOM an
44

55
Use an appropriate dataset such as [cdx-docs](https://huggingface.co/datasets/CycloneDX/cdx-docs) for training and fine-tuning.
66

7+
> NOTE: The tests are unreviewed and non-finalised and therefore must be treated as drafts.
8+
79
## Categories
810

911
The tests include the following categories:
@@ -19,6 +21,8 @@ The tests include the following categories:
1921

2022
System prompt to use Gemini for automated evaluation.
2123

24+
> NOTE: Most LLMs, including Gemini, cannot count the number of entries in a JSON correctly. Plus, the presence of extra quotes and commas can often skew the automated results.
25+
2226
For logic and spec category.
2327

2428
```text

0 commit comments

Comments
 (0)