Add regex unit tests and enable shared linkage in fbcode (#78)

Summary:
Pull Request resolved: #78

As titled. After some investigation it looks like we need `--dynamic-list` to expose the weak symbol `create_fallback_regex` in `regex.so`. Without `--dynamic-list`, linker will only look for locally defined symbol (the weak implementation) and ignore the strong implementation.

This way, when linking `regex_lookahead.so` the exported dynamic symbol can be bound to the strong implementation in `regex_lookahead.cpp`.

This makes the unit tests work in fbcode, where by default we are using `mode/dev` which links everything in shared linkage.

If using `mode/opt` we don't need `--dynamic-list` because the linking order doesn't matter.

Differential Revision: D75391108

Author: larryliu0820

include/pytorch/tokenizers/regex.h

@@ -55,7 +55,7 @@ Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern);
  * @param pattern The regex pattern to compile.
  * @return A unique pointer to an IRegex-compatible object.
  */
-Result<std::unique_ptr<IRegex>> create_fallback_regex(
-    const std::string& pattern) TK_WEAK;
+TK_WEAK Result<std::unique_ptr<IRegex>> create_fallback_regex(
+    const std::string& pattern);
 
 } // namespace tokenizers

src/dynamic_functions_list.txt

@@ -0,0 +1,5 @@
+{
+  extern "C++" {
+    *create_fallback_regex*;
+  };
+};

targets.bzl

@@ -23,6 +23,19 @@ def define_common_targets():
         platforms = PLATFORMS,
     )
 
+    # In fbcode we use a dynamic list to ensure that the dynamic linker doesn't
+    # always bind to the local defined weak symbols. This is telling the linker
+    # to export the list of dynamic symbols that are defined in the library and
+    # when later linked with another shared library, the linker will bind to the
+    # strong symbols in the other library. This is needed in mode/dev where all
+    # libraries are built into shared libraries.
+    # See https://sourceware.org/binutils/docs-2.36/ld/Options.html for more info
+    # on --dynamic-list.
+    runtime.export_file(
+        name = "dynamic_functions_list",
+        src = "src/dynamic_functions_list.txt",
+    )
+
     runtime.cxx_library(
         name = "regex",
         srcs = [
@@ -35,6 +48,9 @@ def define_common_targets():
         exported_external_deps = [
             "re2",
         ],
+        linker_flags = [
+            "--dynamic-list=$(location :dynamic_functions_list)",
+        ],
         visibility = ["//pytorch/tokenizers/..."],
         header_namespace = "",
         platforms = PLATFORMS,
@@ -49,10 +65,12 @@ def define_common_targets():
         ],
         exported_deps = [
             ":headers",
+            ":regex",
         ],
         exported_external_deps = [
             "pcre2",
         ],
+        link_whole = True,
         visibility = [
             "@EXECUTORCH_CLIENTS",
             "//pytorch/tokenizers/...",
@@ -119,6 +137,29 @@ def define_common_targets():
         platforms = PLATFORMS,
     )
 
+    runtime.cxx_library(
+        name = "tiktoken_lookahead",
+        srcs = [
+            "src/tiktoken.cpp",
+        ],
+        deps = [
+            ":regex",
+            ":regex_lookahead",
+        ],
+        exported_deps = [
+            ":bpe_tokenizer_base",
+            ":headers",
+        ],
+        exported_external_deps = [
+            "re2",
+        ],
+        visibility = [
+            "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
+        ],
+        platforms = PLATFORMS,
+    )
+
     runtime.cxx_library(
         name = "hf_tokenizer",
         srcs = [

test/targets.bzl

@@ -96,6 +96,17 @@ def define_common_targets():
         ],
     )
 
+    runtime.cxx_test(
+        name = "test_regex",
+        srcs = [
+            "test_regex.cpp",
+        ],
+        deps = [
+            "//pytorch/tokenizers:regex_lookahead",
+            "//pytorch/tokenizers:headers",
+        ],
+    )
+
     runtime.filegroup(
         name = "resources",
         srcs = native.glob([