add tok

tcapelle · tcapelle · commit 80bd373dfbc0 · 2024-04-12T18:51:10.000+02:00
diff --git a/recipes/configs/llama2/tiny_llama.yaml b/recipes/configs/llama2/tiny_llama.yaml
@@ -5,6 +5,8 @@
 # You can gran the model checkpoint using
 #   wget https://ossci-datasets.s3.amazonaws.com/torchtune/small-ckpt-tune-03082024.pt 
 # place it in the tiny_llama directory.
+# The tokenizer can be downloaded from:
+#   wget https://ossci-datasets.s3.amazonaws.com/torchtune/tokenizer.model
 #
 # and use the same tokenizer as llama2
 #

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,8 @@`
`5`	`5`	`# You can gran the model checkpoint using`
`6`	`6`	`# wget https://ossci-datasets.s3.amazonaws.com/torchtune/small-ckpt-tune-03082024.pt`
`7`	`7`	`# place it in the tiny_llama directory.`
	`8`	`+# The tokenizer can be downloaded from:`
	`9`	`+# wget https://ossci-datasets.s3.amazonaws.com/torchtune/tokenizer.model`
`8`	`10`	`#`
`9`	`11`	`# and use the same tokenizer as llama2`
`10`	`12`	`#`