keep to test tiktoken.h 181 line

yvonwin · Apr 11, 2024 · 7d195e0 · 7d195e0
1 parent 36d2b98
commit 7d195e0
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/tiktoken.h b/tiktoken.h
@@ -179,7 +179,8 @@ class tiktoken {
 
 				if (allowed_special.count(special) == 1) {
 					// return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) };
-					return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) };
+					return { std::move(special), re2::StringPiece(input.data(), input.begin() - start - special.size()) };
+
 				}
 			}