msvc test3

yvonwin · Apr 11, 2024 · 28d651b · 28d651b
1 parent e53b482
commit 28d651b
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/tiktoken.h b/tiktoken.h
@@ -179,9 +179,14 @@ class tiktoken {
 
 				if (allowed_special.count(special) == 1) {
 					#if ! defined(_WIN32)
-						return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) };
+						// return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) };
+						return { std::move(special), re2::StringPiece(reinterpret_cast<const char*>(start), input.begin() - start - special.size()) };
 					#else
-						return { std::move(special), re2::StringPiece(start.data(), input.begin() - start - special.size()) };
+						// build but not work correctly
+						// return { std::move(special), re2::StringPiece(input.data(), input.begin() - start - special.size()) };
+
+						// can work on linux, but msvc?
+						return { std::move(special), re2::StringPiece(reinterpret_cast<const char*>(start), input.begin() - start - special.size()) };
 					#endif
 				}
 			}