make already trained models forward compatible

CogStack · Aug 5, 2024 · b6f0725 · b6f0725
1 parent 3a4b646
commit b6f0725
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/app/model_services/medcat_model_deid.py b/app/model_services/medcat_model_deid.py
@@ -110,6 +110,7 @@ def init_model(self) -> None:
             self._model = self.load_model(self._model_pack_path)
             self._model._addl_ner[0].tokenizer.hf_tokenizer._in_target_context_manager = getattr(self._model._addl_ner[0].tokenizer.hf_tokenizer, "_in_target_context_manager", False)
             self._model._addl_ner[0].tokenizer.hf_tokenizer.clean_up_tokenization_spaces = getattr(self._model._addl_ner[0].tokenizer.hf_tokenizer, "clean_up_tokenization_spaces", None)
+            self._model._addl_ner[0].tokenizer.hf_tokenizer.split_special_tokens = getattr(self._model._addl_ner[0].tokenizer.hf_tokenizer, "split_special_tokens", False)
             if (self._config.DEVICE.startswith("cuda") and torch.cuda.is_available()) or \
                (self._config.DEVICE.startswith("mps") and torch.backends.mps.is_available()) or \
                (self._config.DEVICE.startswith("cpu")):

diff --git a/app/trainers/medcat_deid_trainer.py b/app/trainers/medcat_deid_trainer.py
@@ -95,6 +95,7 @@ def run(trainer: "MedcatDeIdentificationSupervisedTrainer",
                 ner = model._addl_ner[0]
                 ner.tokenizer.hf_tokenizer._in_target_context_manager = getattr(ner.tokenizer.hf_tokenizer, "_in_target_context_manager", False)
                 ner.tokenizer.hf_tokenizer.clean_up_tokenization_spaces = getattr(ner.tokenizer.hf_tokenizer, "clean_up_tokenization_spaces", None)
+                ner.tokenizer.hf_tokenizer.split_special_tokens = getattr(ner.tokenizer.hf_tokenizer, "split_special_tokens", False)
                 _save_pretrained = ner.model.save_pretrained
                 if ("safe_serialization" in inspect.signature(_save_pretrained).parameters):
                     ner.model.save_pretrained = partial(_save_pretrained, safe_serialization=(trainer._config.TRAINING_SAFE_MODEL_SERIALISATION == "true"))
@@ -223,6 +224,7 @@ def run(trainer: "MedcatDeIdentificationSupervisedTrainer",
                 ner = trainer._model_service._model._addl_ner[0]
                 ner.tokenizer.hf_tokenizer._in_target_context_manager = getattr(ner.tokenizer.hf_tokenizer, "_in_target_context_manager", False)
                 ner.tokenizer.hf_tokenizer.clean_up_tokenization_spaces = getattr(ner.tokenizer.hf_tokenizer, "clean_up_tokenization_spaces", None)
+                ner.tokenizer.hf_tokenizer.split_special_tokens = getattr(ner.tokenizer.hf_tokenizer, "split_special_tokens", False)
                 eval_results, examples = ner.eval(data_file.name)
                 cui2names = {}
                 eval_results.sort_values(by=["cui"])