From 90447bc9937b6c97960bb40d93922feb18087782 Mon Sep 17 00:00:00 2001 From: Laurent Mazare Date: Fri, 9 Feb 2024 17:36:50 +0100 Subject: [PATCH] Add the custom tokenizer. (#1686) --- candle-examples/examples/chatglm/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/candle-examples/examples/chatglm/main.rs b/candle-examples/examples/chatglm/main.rs index 495588f1f5..ae86418165 100644 --- a/candle-examples/examples/chatglm/main.rs +++ b/candle-examples/examples/chatglm/main.rs @@ -202,7 +202,9 @@ fn main() -> Result<()> { let repo = api.repo(Repo::with_revision(model_id, RepoType::Model, revision)); let tokenizer_filename = match args.tokenizer { Some(file) => std::path::PathBuf::from(file), - None => repo.get("tokenizer.json")?, + None => api + .model("lmz/candle-chatglm".to_string()) + .get("chatglm-tokenizer.json")?, }; let filenames = match args.weight_file { Some(weight_file) => vec![std::path::PathBuf::from(weight_file)],