From fa100121a29dbb8ec2885be59e00994d918e724e Mon Sep 17 00:00:00 2001 From: yvonwin Date: Thu, 11 Apr 2024 16:52:43 +0800 Subject: [PATCH] update version --- third_party/abseil-cpp | 2 +- third_party/ggml | 2 +- tiktoken.h | 9 ++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index 0378614..8f9e5f0 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit 03786143361db9d8581cb02ed1c5027d732b62fc +Subproject commit 8f9e5f0203c8a708c71f4498bf59355d1d7bbf87 diff --git a/third_party/ggml b/third_party/ggml index 9a867f1..f841cd0 160000 --- a/third_party/ggml +++ b/third_party/ggml @@ -1 +1 @@ -Subproject commit 9a867f1a56187e168a222078595c00ca8f1ed2af +Subproject commit f841cd014d1ea0904a9ec733a3c8ee3a5171e0ac diff --git a/tiktoken.h b/tiktoken.h index bec5178..5096c9d 100644 --- a/tiktoken.h +++ b/tiktoken.h @@ -177,15 +177,10 @@ class tiktoken { break; } - if (allowed_special.count(special) == 1) { - #if ! defined(_WIN32) - return { std::move(special), re2::StringPiece(start, input.begin() - start - special.size()) }; // failed on windows building - #else - return { std::move(special), re2::StringPiece(input.data(), input.begin() - start - special.size()) }; // TODO test msvc use - #endif + if (allowed_special.count(special) == 1) { + return { std::move(special), re2::StringPiece(&*start, input.begin() - start - special.size()) }; // failed on windows building } } - return { std::nullopt, input }; }