enable skipping_special_tokens in hf_tokenizer detokenize()

Author: benniekiss

llama_cpp/llama_tokenizer.py

@@ -78,18 +78,19 @@ def tokenize(
         )
 
     def detokenize(
-        self, tokens: List[int], prev_tokens: Optional[List[int]] = None
+        self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = True
     ) -> bytes:
+        skip_special_tokens = not special 
         if prev_tokens is not None:
-            text = self.hf_tokenizer.decode(prev_tokens + tokens).encode(
+            text = self.hf_tokenizer.decode(prev_tokens + tokens, skip_special_tokens=skip_special_tokens).encode(
                 "utf-8", errors="ignore"
             )
-            prev_text = self.hf_tokenizer.decode(prev_tokens).encode(
+            prev_text = self.hf_tokenizer.decode(prev_tokens, skip_special_tokens=skip_special_tokens).encode(
                 "utf-8", errors="ignore"
             )
             return text[len(prev_text) :]
         else:
-            return self.hf_tokenizer.decode(tokens).encode("utf-8", errors="ignore")
+            return self.hf_tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens).encode("utf-8", errors="ignore")
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer":