Skip to content

Commit 4e5b81b

Browse files
authored
Fix Llama 3.2 tokenizer (#1772)
1 parent c16c72c commit 4e5b81b

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

litgpt/tokenizer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def check_if_bos_token_used(self, checkpoint_dir: Path) -> bool:
9494
config = json.load(fp)
9595
# for LlaMA-3 tokenizer there is no `add_bos_token` at all and `tokenizer_class` is only
9696
# `PreTrainedTokenizerFast`
97-
if checkpoint_dir.stem.startswith("Meta-Llama-3"):
97+
if checkpoint_dir.stem.startswith(("Meta-Llama-3", "Llama-3")):
9898
return True
9999
if "add_bos_token" in config:
100100
return config["add_bos_token"]

0 commit comments

Comments
 (0)