diff --git a/lucidia_llm/tokenizer/spm.yaml b/lucidia_llm/tokenizer/spm.yaml new file mode 100644 index 0000000..72b560f --- /dev/null +++ b/lucidia_llm/tokenizer/spm.yaml @@ -0,0 +1,3 @@ +# SentencePiece model configuration for Lucidia +model_type: bpe +vocab_size: 50000