chore!: upgrade Ray pins and pyarrow pins (#3612)

Updates the lower bound of pyarrow to `pyarrow>=8.0.0`.

This allows us to flatten some code checks.

However, it turns out that our tests aren't being properly skipped -- I
had to update the tests to just skip based on our lower bound (skip if
version < 9.0.0) which very loose, but otherwise searching for the
individual versions for each suite of tests was quite difficult.

---------

Co-authored-by: Jay Chia <jaychia94@gmail.com@users.noreply.github.com>

Author: jaychia

.github/workflows/python-package.yml

@@ -26,26 +26,26 @@ jobs:
       matrix:
         python-version: ['3.9', '3.10']
         daft-runner: [py, ray, native]
-        pyarrow-version: [7.0.0, 16.0.0]
+        pyarrow-version: [8.0.0, 16.0.0]
         os: [ubuntu-20.04, windows-latest]
         exclude:
         - daft-runner: ray
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
         - daft-runner: py
           python-version: '3.10'
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
         - daft-runner: native
           python-version: '3.10'
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
         - python-version: '3.9'
           pyarrow-version: 16.0.0
         - os: windows-latest
           python-version: '3.9'
         - os: windows-latest
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
     steps:
     - uses: actions/checkout@v4
     - uses: moonrepo/setup-rust@v1
@@ -93,7 +93,7 @@ jobs:
       run: uv pip install pyarrow==${{ matrix.pyarrow-version }}
 
     - name: Override deltalake for pyarrow
-      if: ${{ (matrix.pyarrow-version == '7.0.0') }}
+      if: ${{ (matrix.pyarrow-version == '8.0.0') }}
       run: uv pip install deltalake==0.10.0
 
     - name: Build library and Test with pytest (unix)

daft/table/table_io.py

@@ -554,16 +554,11 @@ def _write_tabular_arrow_table(
 ):
     kwargs = dict()
 
-    from daft.utils import get_arrow_version
+    kwargs["max_rows_per_file"] = rows_per_file
+    kwargs["min_rows_per_group"] = rows_per_row_group
+    kwargs["max_rows_per_group"] = rows_per_row_group
 
-    arrow_version = get_arrow_version()
-
-    if arrow_version >= (7, 0, 0):
-        kwargs["max_rows_per_file"] = rows_per_file
-        kwargs["min_rows_per_group"] = rows_per_row_group
-        kwargs["max_rows_per_group"] = rows_per_row_group
-
-    if arrow_version >= (8, 0, 0) and not create_dir:
+    if not create_dir:
         kwargs["create_dir"] = False
 
     basename_template = _generate_basename_template(format.default_extname, version)

pyproject.toml

@@ -5,7 +5,7 @@ requires = ["maturin>=1.5.0,<2.0.0"]
 [project]
 authors = [{name = "Eventual Inc", email = "daft@eventualcomputing.com"}]
 dependencies = [
-  "pyarrow >= 7.0.0",
+  "pyarrow >= 8.0.0",
   "fsspec",
   "tqdm",
   "typing-extensions >= 4.0.0; python_version < '3.10'"

tests/integration/iceberg/conftest.py

@@ -10,8 +10,10 @@
 
 pyiceberg = pytest.importorskip("pyiceberg")
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
-pytestmark = pytest.mark.skipif(PYARROW_LE_8_0_0, reason="iceberg writes only supported if pyarrow >= 8.0.0")
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
+pytestmark = pytest.mark.skipif(
+    PYARROW_LOWER_BOUND_SKIP, reason="iceberg writes not supported on old versions of pyarrow"
+)
 
 import tenacity
 from pyiceberg.catalog import Catalog, load_catalog

tests/io/delta_lake/test_table_read.py

@@ -8,10 +8,10 @@
 from daft.logical.schema import Schema
 from tests.utils import assert_pyarrow_tables_equal
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
 pytestmark = pytest.mark.skipif(
-    PYARROW_LE_8_0_0,
-    reason="deltalake only supported if pyarrow >= 8.0.0",
+    PYARROW_LOWER_BOUND_SKIP,
+    reason="deltalake not supported on older versions of pyarrow",
 )
 
 

tests/io/delta_lake/test_table_read_pushdowns.py

@@ -16,10 +16,10 @@
 from daft.logical.schema import Schema
 from tests.utils import assert_pyarrow_tables_equal
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
 pytestmark = pytest.mark.skipif(
-    PYARROW_LE_8_0_0,
-    reason="deltalake only supported if pyarrow >= 8.0.0",
+    PYARROW_LOWER_BOUND_SKIP,
+    reason="deltalake not supported on older versions of pyarrow",
 )
 
 

tests/io/delta_lake/test_table_write.py

@@ -12,10 +12,10 @@
 from daft.logical.schema import Schema
 from tests.conftest import get_tests_daft_runner_name
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
 pytestmark = pytest.mark.skipif(
-    PYARROW_LE_8_0_0,
-    reason="deltalake only supported if pyarrow >= 8.0.0",
+    PYARROW_LOWER_BOUND_SKIP,
+    reason="deltalake not supported on older versions of pyarrow",
 )
 
 

tests/io/hudi/test_table_read.py

@@ -7,8 +7,8 @@
 
 import daft
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
-pytestmark = pytest.mark.skipif(PYARROW_LE_8_0_0, reason="hudi only supported if pyarrow >= 8.0.0")
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
+pytestmark = pytest.mark.skipif(PYARROW_LOWER_BOUND_SKIP, reason="hudi not supported on old versions of pyarrow")
 
 
 def test_read_table(get_testing_table_for_supported_cases):

tests/io/iceberg/test_iceberg_writes.py

@@ -10,8 +10,8 @@
 
 pyiceberg = pytest.importorskip("pyiceberg")
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
-pytestmark = pytest.mark.skipif(PYARROW_LE_8_0_0, reason="iceberg only supported if pyarrow >= 8.0.0")
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
+pytestmark = pytest.mark.skipif(PYARROW_LOWER_BOUND_SKIP, reason="iceberg not supported on old versions of pyarrow")
 
 
 from pyiceberg.catalog.sql import SqlCatalog

tests/io/lancedb/test_lancedb_reads.py

@@ -11,8 +11,8 @@
     "long": [-122.7, -74.1],
 }
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
-pytestmark = pytest.mark.skipif(PYARROW_LE_8_0_0, reason="lance only supported if pyarrow >= 8.0.0")
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
+pytestmark = pytest.mark.skipif(PYARROW_LOWER_BOUND_SKIP, reason="lance not supported on old versions of pyarrow")
 
 
 @pytest.fixture(scope="function")

tests/io/lancedb/test_lancedb_writes.py

@@ -12,9 +12,8 @@
     "long": [-122.7, -74.1],
 }
 
-PYARROW_LE_8_0_0 = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (8, 0, 0)
-
-pytestmark = pytest.mark.skipif(PYARROW_LE_8_0_0, reason="lance only supported if pyarrow >= 8.0.0")
+PYARROW_LOWER_BOUND_SKIP = tuple(int(s) for s in pa.__version__.split(".") if s.isnumeric()) < (9, 0, 0)
+pytestmark = pytest.mark.skipif(PYARROW_LOWER_BOUND_SKIP, reason="lance not supported on old versions of pyarrow")
 
 
 @pytest.fixture(scope="function")