1 file changed
+1
-1
lines changedSubmodule tokenizers updated 17 files
- CMakeLists.txt+27-31
- Utils.cmake+50
- include/pytorch/tokenizers/bpe_tokenizer_base.h+21
- include/pytorch/tokenizers/error.h+3
- include/pytorch/tokenizers/pcre2_regex.h+8-6
- include/pytorch/tokenizers/re2_regex.h+8-6
- include/pytorch/tokenizers/regex.h+17-1
- include/pytorch/tokenizers/std_regex.h+8-4
- src/hf_tokenizer.cpp+15-2
- src/pcre2_regex.cpp+12-7
- src/re2_regex.cpp+15-1
- src/regex.cpp+33-50
- src/regex_lookahead.cpp+50
- src/std_regex.cpp+12-1
- src/tiktoken.cpp+1-17
- targets.bzl+17-29
- test/targets.bzl+11
0 commit comments