msvc test7: try claude twice

yvonwin · Apr 11, 2024 · 2eeed9f · 2eeed9f
1 parent a8f447d
commit 2eeed9f
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/tiktoken.h b/tiktoken.h
@@ -171,6 +171,7 @@ class tiktoken {
 			if (special_regex_ == nullptr) return { std::nullopt, input };
 
 			auto start = input.begin();
+			size_t start_len = input.size(); // Store the initial length
 			std::string special;
 			while (true) {
 				if (!re2::RE2::FindAndConsume(&input, *special_regex_, &special)) {
@@ -190,7 +191,7 @@ class tiktoken {
 						// return { std::move(special), re2::StringPiece(reinterpret_cast<const char*>(start), input.begin() - start - special.size()) };
 						// return { std::move(special), re2::StringPiece(reinterpret_cast<const char*>(start), static_cast<size_t>(input.begin() - start - special.size())) };
 						// return { special, re2::StringPiece(start, input.begin() - start - special.size()) };
-						return { special, re2::StringPiece(start, input.begin() - start - special.size()) };
+						return { special, re2::StringPiece(start, start_len - input.size() - special.size()) };
 					#endif
 				}
 			}