enable detokenizing special tokens

Author: benniekiss

llama_cpp/_internals.py

@@ -214,7 +214,7 @@ def token_to_piece(self, token: int, special: bool = False) -> bytes:
         llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special)
         return bytes(buf)
 
-    def detokenize(self, tokens: List[int], special: bool = False) -> bytes:
+    def detokenize(self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False) -> bytes:
         assert self.model is not None
         output = b""
         size = 32

llama_cpp/llama.py

@@ -580,7 +580,7 @@ def tokenize(
         return self.tokenizer_.tokenize(text, add_bos, special)
 
     def detokenize(
-        self, tokens: List[int], prev_tokens: Optional[List[int]] = None
+        self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False
     ) -> bytes:
         """Detokenize a list of tokens.
 
@@ -591,7 +591,7 @@ def detokenize(
         Returns:
             The detokenized string.
         """
-        return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens)
+        return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens, special=special)
 
     def set_cache(self, cache: Optional[BaseLlamaCache]):
         """Set the cache.

llama_cpp/llama_tokenizer.py

@@ -26,7 +26,7 @@ def tokenize(
 
     @abc.abstractmethod
     def detokenize(
-        self, tokens: List[int], prev_tokens: Optional[List[int]] = None
+        self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = True
     ) -> bytes:
         """Detokenize the tokens into text.
 
@@ -47,9 +47,9 @@ def tokenize(
         return self._model.tokenize(text, add_bos=add_bos, special=special)
 
     def detokenize(
-        self, tokens: List[int], prev_tokens: Optional[List[int]] = None
+        self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = True
     ) -> bytes:
-        return self._model.detokenize(tokens)
+        return self._model.detokenize(tokens, prev_tokens=prev_tokens, special=special)
 
     def encode(
         self, text: str, add_bos: bool = True, special: bool = True