Add the custom tokenizer. (#1686)

huggingface · Feb 9, 2024 · 90447bc · 90447bc
1 parent 40ce160
commit 90447bc
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/candle-examples/examples/chatglm/main.rs b/candle-examples/examples/chatglm/main.rs
@@ -202,7 +202,9 @@ fn main() -> Result<()> {
     let repo = api.repo(Repo::with_revision(model_id, RepoType::Model, revision));
     let tokenizer_filename = match args.tokenizer {
         Some(file) => std::path::PathBuf::from(file),
-        None => repo.get("tokenizer.json")?,
+        None => api
+            .model("lmz/candle-chatglm".to_string())
+            .get("chatglm-tokenizer.json")?,
     };
     let filenames = match args.weight_file {
         Some(weight_file) => vec![std::path::PathBuf::from(weight_file)],