feat: add a way to count tokens without encoding the whole text

- adds highly performant `isWithinTokenLimit` to count tokens without encoding the whole text
- improve overall performance by removing transitive arrays
- include precomputed `bpeRanks`
- add type-checking
- fix a few minor bugs (thanks to type-checking)
- add generator versions of both decoder and encoder

Author: niieani

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.npmrc
+*.d.ts
+.npmrc

.npmignore

@@ -0,0 +1,2 @@
+.npmrc
+tsconfig.json

Encoder.js

@@ -0,0 +1,20 @@
+const path = require("path");
+const fs = require("fs");
+const { dictZip, range } = require("../utils");
+
+const bpe_file = fs.readFileSync(path.join(__dirname, "./vocab.bpe"), "utf-8");
+const lines = bpe_file.split("\n");
+
+const bpe_merges = lines.slice(1, lines.length - 1).map((x) =>
+  x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(","),
+);
+
+const bpe_ranks = dictZip(bpe_merges, range(0, bpe_merges.length));
+
+fs.writeFileSync(
+  path.join(__dirname, "./bpe_ranks.json"),
+  JSON.stringify(bpe_ranks),
+);