process prev_tokens

Author: benniekiss

llama_cpp/_internals.py

@@ -214,7 +214,7 @@ def token_to_piece(self, token: int, special: bool = False) -> bytes:
         llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special)
         return bytes(buf)
 
-    def detokenize(self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False) -> bytes:
+    def detokenize(self, tokens: List[int], special: bool = False) -> bytes:
         assert self.model is not None
         output = b""
         size = 32

llama_cpp/llama_tokenizer.py

@@ -49,7 +49,12 @@ def tokenize(
     def detokenize(
         self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = True
     ) -> bytes:
-        return self._model.detokenize(tokens, prev_tokens=prev_tokens, special=special)
+        if prev_tokens is not None:
+            text = self._model.detokenize(prev_tokens + tokens, special=special)
+            prev_text = self._model.detokenize(prev_tokens, special=special)
+            return text[len(prev_text) :]
+        else:
+            return self._model.detokenize(tokens, special=special)
 
     def encode(
         self, text: str, add_bos: bool = True, special: bool = True