From 28d651b909080bb094ae60498192350328c41815 Mon Sep 17 00:00:00 2001 From: yvonwin Date: Thu, 11 Apr 2024 12:49:50 +0800 Subject: [PATCH] msvc test3 --- tiktoken.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tiktoken.h b/tiktoken.h index 1878f5a..33619fa 100644 --- a/tiktoken.h +++ b/tiktoken.h @@ -179,9 +179,14 @@ class tiktoken { if (allowed_special.count(special) == 1) { #if ! defined(_WIN32) - return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) }; + // return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) }; + return { std::move(special), re2::StringPiece(reinterpret_cast(start), input.begin() - start - special.size()) }; #else - return { std::move(special), re2::StringPiece(start.data(), input.begin() - start - special.size()) }; + // build but not work correctly + // return { std::move(special), re2::StringPiece(input.data(), input.begin() - start - special.size()) }; + + // can work on linux, but msvc? + return { std::move(special), re2::StringPiece(reinterpret_cast(start), input.begin() - start - special.size()) }; #endif } }