diff --git a/tiktoken.h b/tiktoken.h index 1878f5a..33619fa 100644 --- a/tiktoken.h +++ b/tiktoken.h @@ -179,9 +179,14 @@ class tiktoken { if (allowed_special.count(special) == 1) { #if ! defined(_WIN32) - return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) }; + // return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) }; + return { std::move(special), re2::StringPiece(reinterpret_cast(start), input.begin() - start - special.size()) }; #else - return { std::move(special), re2::StringPiece(start.data(), input.begin() - start - special.size()) }; + // build but not work correctly + // return { std::move(special), re2::StringPiece(input.data(), input.begin() - start - special.size()) }; + + // can work on linux, but msvc? + return { std::move(special), re2::StringPiece(reinterpret_cast(start), input.begin() - start - special.size()) }; #endif } }