Upgrading to rs-0.35 (#144)

* Upgrading to rs-0.35 * Upgrading TS, fixing tests * Upgrading napi, adding tests * Fixing linting error
pola-rs · Dec 5, 2023 · 8389398 · 8389398
1 parent 14a8fb4
commit 8389398
Show file tree

Hide file tree

Showing 22 changed files with 259 additions and 179 deletions.
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -14,7 +14,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2023-07-27
+          toolchain: nightly-2023-11-15
           components: rustfmt, clippy
       - name: Install ghp-import
         uses: actions/setup-python@v4

diff --git a/.github/workflows/test-js.yaml b/.github/workflows/test-js.yaml
@@ -18,7 +18,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2023-07-27
+          toolchain: nightly-2023-11-15
           components: rustfmt, clippy
       - run: yarn --version
       - name: Install Node Dependencies
@@ -43,7 +43,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2023-07-27
+          toolchain: nightly-2023-11-15
           components: rustfmt, clippy
       - name: Bun version
         uses: oven-sh/setup-bun@v1

diff --git a/Cargo.toml b/Cargo.toml
@@ -15,13 +15,13 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 ahash = "0.8.3"
 bincode = "1.3.3"
-napi = {version = "2.13.3", default-features = false, features = ["napi8", "serde-json", "experimental"]}
-napi-derive = {version = "2.13.0", default-features = false}
-polars-core = {git = "https://github.com/pola-rs/polars.git", rev = "60adaef43e1b3306d5da8cf736accea2e3f03d9b", default-features = false}
-polars-io = {git = "https://github.com/pola-rs/polars.git", rev = "60adaef43e1b3306d5da8cf736accea2e3f03d9b", default-features = false}
-polars-lazy = {git = "https://github.com/pola-rs/polars.git", rev = "60adaef43e1b3306d5da8cf736accea2e3f03d9b", default-features = false}
+napi = {version = "2.14.1", default-features = false, features = ["napi8", "serde-json"]}
+napi-derive = {version = "2.14.2", default-features = false}
+polars-core = {git = "https://github.com/pola-rs/polars.git", rev = "b13afbecac039205dacbaca766ecca4bf441b347", default-features = false}
+polars-io = {git = "https://github.com/pola-rs/polars.git", rev = "b13afbecac039205dacbaca766ecca4bf441b347", default-features = false}
+polars-lazy = {git = "https://github.com/pola-rs/polars.git", rev = "b13afbecac039205dacbaca766ecca4bf441b347", default-features = false}
 thiserror = "1"
-smartstring = { version = "1" }
+smartstring = {version = "1"}
 serde_json = {version = "1"}
 either = "1.9"
 
@@ -52,7 +52,6 @@ features = [
   "concat_str",
   "row_hash",
   "reinterpret",
-  "decompress-fast",
   "mode",
   "extract_jsonpath",
   "lazy_regex",
@@ -66,8 +65,6 @@ features = [
   "diff",
   "pct_change",
   "moment",
-  "true_div",
-  "dtype-categorical",
   "diagonal_concat",
   "horizontal_concat",
   "abs",
@@ -91,13 +88,14 @@ features = [
   "arg_where",
   "timezones",
   "peaks",
-  "string_justify",
+  "string_pad",
+  "cov"
 ]
 git = "https://github.com/pola-rs/polars.git"
-rev = "60adaef43e1b3306d5da8cf736accea2e3f03d9b"
+rev = "b13afbecac039205dacbaca766ecca4bf441b347"
 
 [build-dependencies]
-napi-build = "2.0.1"
+napi-build = "2.1.0"
 
 [profile.release]
 codegen-units = 1

diff --git a/__tests__/dataframe.test.ts b/__tests__/dataframe.test.ts
@@ -1539,7 +1539,7 @@ describe("io", () => {
   test("writeCSV:string:header", () => {
     const actual = df
       .clone()
-      .writeCSV({ sep: "X", hasHeader: false })
+      .writeCSV({ sep: "X", includeHeader: false })
       .toString();
     const expected = "1X6\n2X2\n9X8\n";
     expect(actual).toEqual(expected);

diff --git a/__tests__/expr.test.ts b/__tests__/expr.test.ts
@@ -331,8 +331,8 @@ describe("expr", () => {
   });
   test.each`
     args                   | hashValue
-    ${[0]}                 | ${6574965099265562227n}
-    ${[{ k0: 1n, k1: 1 }]} | ${6574965099265562227n}
+    ${[0]}                 | ${6340063056640878722n}
+    ${[{ k0: 1n, k1: 1 }]} | ${9788354747012366704n}
   `("$# hash", ({ args, hashValue }) => {
     const df = pl.DataFrame({ a: [1] });
     const expected = pl.DataFrame({ hash: [hashValue] });
@@ -863,19 +863,19 @@ describe("expr", () => {
       "take:list": [1, 2, 2, 3],
     });
     const actual = df.select(
-      col("a").take([0, 2, 3, 5]).as("take:array"),
+      col("a").gather([0, 2, 3, 5]).as("take:array"),
       col("a")
-        .take(lit([0, 1, 2, 3]))
+        .gather(lit([0, 1, 2, 3]))
         .as("take:list"),
     );
     expect(actual).toFrameEqual(expected);
   });
-  test("takeEvery", () => {
+  test("gatherEvery", () => {
     const df = pl.DataFrame({ a: [1, 1, 2, 2, 3, 3, 8, null, 1] });
     const expected = pl.DataFrame({
       everyother: [1, 2, 3, 8, 1],
     });
-    const actual = df.select(col("a").takeEvery(2).as("everyother"));
+    const actual = df.select(col("a").gatherEvery(2).as("everyother"));
     expect(actual).toFrameEqual(expected);
   });
   test("unique", () => {
@@ -1391,6 +1391,21 @@ describe("expr.str", () => {
   });
 });
 describe("expr.lst", () => {
+  test("argMax", () => {
+    const s0 = pl.Series("a", [[1, 2, 3]]);
+    let actual = s0.lst.argMax();
+    let expected = pl.Series("a", [2]);
+    expect(actual.seriesEqual(expected));
+    actual = s0.lst.argMin();
+    expected = pl.Series("a", [0]);
+    expect(actual.seriesEqual(expected));
+  });
+  test("contains", () => {
+    const s0 = pl.Series("a", [[1, 2]]);
+    const actual = s0.lst.contains(1);
+    const expected = pl.Series("a", [true]);
+    expect(actual.seriesEqual(expected));
+  });
   test("concat", () => {
     const s0 = pl.Series("a", [[1, 2]]);
     const s1 = pl.Series("b", [[3, 4, 5]]);
@@ -1422,6 +1437,12 @@ describe("expr.lst", () => {
         .seriesEqual(expected),
     );
   });
+  test("diff", () => {
+    const s0 = pl.Series("a", [[1, 2, 3]]);
+    const actual = s0.lst.diff();
+    const expected = pl.Series("a", [null, 1, 1]);
+    expect(actual.seriesEqual(expected));
+  });
   test("get", () => {
     const df = pl.DataFrame({ a: [[1, 10, 11], [2, 10, 12], [1]] });
     const expected = pl.DataFrame({ get: [11, 12, null] });
@@ -1448,6 +1469,12 @@ describe("expr.lst", () => {
     expect(actual).toFrameEqual(expected);
     expect(actualFromSeries).toFrameEqual(expected);
   });
+  test("eval", () => {
+    const s0 = pl.Series("a", [[3, 5, 6]]);
+    const actual = s0.lst.eval(pl.element().rank());
+    const expected = pl.Series("a", [1, 2, 3]);
+    expect(actual.seriesEqual(expected));
+  });
   test("first", () => {
     const df = pl.DataFrame({
       a: [
@@ -1466,6 +1493,30 @@ describe("expr.lst", () => {
     expect(actual).toFrameEqual(expected);
     expect(actualFromSeries).toFrameEqual(expected);
   });
+  test("head", () => {
+    const s0 = pl.Series("a", [[3, 5, 6, 7, 8]]);
+    let actual = s0.lst.head(1);
+    let expected = pl.Series("a", [3]);
+    expect(actual.seriesEqual(expected));
+    actual = s0.lst.head();
+    expected = pl.Series("a", [3, 5, 6, 7, 8]);
+    expect(actual.seriesEqual(expected));
+  });
+  test("tail", () => {
+    const s0 = pl.Series("a", [[3, 5, 6, 7, 8]]);
+    let actual = s0.lst.tail(1);
+    let expected = pl.Series("a", [8]);
+    expect(actual.seriesEqual(expected));
+    actual = s0.lst.tail();
+    expected = pl.Series("a", [3, 5, 6, 7, 8]);
+    expect(actual.seriesEqual(expected));
+  });
+  test("shift", () => {
+    const s0 = pl.Series("a", [[3, 5, 6]]);
+    const actual = s0.lst.shift(-1);
+    const expected = pl.Series("a", [5, 6, null]);
+    expect(actual.seriesEqual(expected));
+  });
   test("join", () => {
     const df = pl.DataFrame({ a: [["ab", "cd"], ["e", "fg"], ["h"]] });
     const expected = pl.DataFrame({ joinedString: ["ab,cd", "e,fg", "h"] });
@@ -1630,16 +1681,13 @@ describe("expr.lst", () => {
       col("a").lst.sort().as("sort"),
       col("a").lst.sort({ reverse: true }).as("sort:reverse"),
     );
-
     const sortSeries = df.getColumn("a").lst.sort().rename("sort");
-
     const sortReverseSeries = df
       .getColumn("a")
-      .lst.sort(true)
+      .lst.sort({ reverse: true })
       .rename("sort:reverse");
 
     const actualFromSeries = pl.DataFrame([sortSeries, sortReverseSeries]);
-
     expect(actual).toFrameEqual(expected);
     expect(actualFromSeries).toFrameEqual(expected);
     expect(actualFromSeries).toFrameEqual(actual);

diff --git a/__tests__/lazyframe.test.ts b/__tests__/lazyframe.test.ts
@@ -27,18 +27,30 @@ describe("lazyframe", () => {
     const actual = await expected.lazy().collect();
     expect(actual).toFrameEqual(expected);
   });
-
   test("describeOptimizedPlan", () => {
     const df = pl
       .DataFrame({
         foo: [1, 2],
         bar: ["a", "b"],
       })
       .lazy();
-    const actual = df.describeOptimizedPlan().replace(/\s+/g, " ");
-    expect(actual).toEqual(
-      `DF ["foo", "bar"]; PROJECT */2 COLUMNS; SELECTION: "None"`,
-    );
+    let actual = df.describeOptimizedPlan().replace(/\s+/g, " ");
+    const expected = `DF ["foo", "bar"]; PROJECT */2 COLUMNS; SELECTION: "None"`;
+    expect(actual).toEqual(expected);
+    actual = df.describePlan().replace(/\s+/g, " ");
+    expect(actual).toEqual(expected);
+  });
+  test("cache", () => {
+    const df = pl.DataFrame({
+      foo: [1, 2, 3],
+    });
+    const expected = pl.DataFrame({
+      foo: [1, 2, 3],
+    });
+    let actual = df.lazy().cache().collectSync();
+    expect(actual).toFrameEqual(expected);
+    actual = df.lazy().clone().collectSync();
+    expect(actual).toFrameEqual(expected);
   });
   test("drop", () => {
     const df = pl.DataFrame({
@@ -240,12 +252,10 @@ describe("lazyframe", () => {
       .lazy()
       .explode("list_1")
       .collectSync();
-
     const expected = pl.DataFrame({
       letters: ["c", "c", "a", "a"],
       list_1: [1, 2, 1, 3],
     });
-
     expect(actual).toFrameEqualIgnoringOrder(expected);
   });
   test("fetch", async () => {

diff --git a/__tests__/series.test.ts b/__tests__/series.test.ts
@@ -409,8 +409,8 @@ describe("series", () => {
     ${numSeries()}  | ${"sort"}            | ${[{ reverse: false }]}
     ${numSeries()}  | ${"sum"}             | ${[]}
     ${numSeries()}  | ${"tail"}            | ${[]}
-    ${numSeries()}  | ${"take"}            | ${[[1, 2]]}
-    ${numSeries()}  | ${"takeEvery"}       | ${[1]}
+    ${numSeries()}  | ${"gather"}          | ${[[1, 2]]}
+    ${numSeries()}  | ${"gatherEvery"}     | ${[1]}
     ${numSeries()}  | ${"toArray"}         | ${[]}
     ${numSeries()}  | ${"unique"}          | ${[]}
     ${numSeries()}  | ${"valueCounts"}     | ${[]}
@@ -472,7 +472,7 @@ describe("series", () => {
     ${"getIndex"}        | ${pl.Series(["a", "b", "c"]).getIndex(0)}                                                  | ${"a"}
     ${"hasValidity"}     | ${pl.Series([1, null, 2]).hasValidity()}                                                   | ${true}
     ${"hasValidity"}     | ${pl.Series([1, 1, 2]).hasValidity()}                                                      | ${false}
-    ${"hash"}            | ${pl.Series([1]).hash()}                                                                   | ${pl.Series([6574965099265562227n])}
+    ${"hash"}            | ${pl.Series([1]).hash()}                                                                   | ${pl.Series([6340063056640878722n])}
     ${"head"}            | ${pl.Series([1, 2, 3, 4, 5, 5, 5]).head()}                                                 | ${pl.Series([1, 2, 3, 4, 5])}
     ${"head"}            | ${pl.Series([1, 2, 3, 4, 5, 5, 5]).head(2)}                                                | ${pl.Series([1, 2])}
     ${"interpolate"}     | ${pl.Series([1, 2, null, null, 5]).interpolate()}                                          | ${pl.Series([1, 2, 3, 4, 5])}
@@ -523,8 +523,8 @@ describe("series", () => {
     ${"sort"}            | ${pl.Series([4, 2, 5, 0]).sort({ reverse: false })}                                        | ${pl.Series([0, 2, 4, 5])}
     ${"sum"}             | ${pl.Series([1, 2, 2, 1]).sum()}                                                           | ${6}
     ${"tail"}            | ${pl.Series([1, 2, 2, 1]).tail(2)}                                                         | ${pl.Series([2, 1])}
-    ${"takeEvery"}       | ${pl.Series([1, 3, 2, 9, 1]).takeEvery(2)}                                                 | ${pl.Series([1, 2, 1])}
-    ${"take"}            | ${pl.Series([1, 3, 2, 9, 1]).take([0, 1, 3])}                                              | ${pl.Series([1, 3, 9])}
+    ${"gatherEvery"}     | ${pl.Series([1, 3, 2, 9, 1]).gatherEvery(2)}                                               | ${pl.Series([1, 2, 1])}
+    ${"gather"}          | ${pl.Series([1, 3, 2, 9, 1]).gather([0, 1, 3])}                                            | ${pl.Series([1, 3, 9])}
     ${"toArray"}         | ${pl.Series([1, 2, 3]).toArray()}                                                          | ${[1, 2, 3]}
     ${"unique"}          | ${pl.Series([1, 2, 3, 3]).unique().sort()}                                                 | ${pl.Series([1, 2, 3])}
     ${"cumCount"}        | ${pl.Series([1, 2, 3, 3]).cumCount()}                                                      | ${pl.Series([0, 1, 2, 3])}

diff --git a/package.json b/package.json
@@ -56,19 +56,19 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^1.3.3",
-    "@napi-rs/cli": "^2.16.4",
-    "@types/chance": "^1.1.5",
-    "@types/jest": "^29.5.7",
-    "@types/node": "^20.8.10",
+    "@napi-rs/cli": "^2.16.5",
+    "@types/chance": "^1.1.6",
+    "@types/jest": "^29.5.10",
+    "@types/node": "^20.10.0",
     "chance": "^1.1.11",
     "jest": "^29.7.0",
     "source-map-support": "^0.5.21",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "typedoc": "^0.25.3",
-    "typescript": "5.2.2"
+    "typescript": "5.3.2"
   },
-  "packageManager": "yarn@3.6.2",
+  "packageManager": "yarn@4.0.2",
   "workspaces": [
     "benches"
   ]