From 516db43d9be7892b7cad159a827b99d58705ba88 Mon Sep 17 00:00:00 2001 From: Jack <32371937+jackzhxng@users.noreply.github.com> Date: Tue, 13 May 2025 16:36:11 -0700 Subject: [PATCH] Revert "Handle null bos and eos token (#66)" This reverts commit c11169ce9efc11dd0e692a639e7bcbfd75818d4b. --- src/hf_tokenizer.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/hf_tokenizer.cpp b/src/hf_tokenizer.cpp index a04005b..47dbb4f 100644 --- a/src/hf_tokenizer.cpp +++ b/src/hf_tokenizer.cpp @@ -147,15 +147,8 @@ Error HFTokenizer::load(const std::string& path) { // Pull out the token strings try { - const std::string bos_token = - parsed_config_json.contains("bos_token") && !parsed_config_json["bos_token"].is_null() - ? parsed_config_json["bos_token"].get() - : ""; - - const std::string eos_token = - parsed_config_json.contains("eos_token") && !parsed_config_json["eos_token"].is_null() - ? parsed_config_json["eos_token"].get() - : ""; + const std::string bos_token = parsed_config_json.at("bos_token"); + const std::string eos_token = parsed_config_json.at("eos_token"); const auto bos_res = special_token_map_->tryGetInteger(bos_token); const auto eos_res = special_token_map_->tryGetInteger(eos_token); if (!bos_res) {