[Data] remove pyarrow 8 check for arrow.py (#52404)

<!-- Thank you for your contribution! Please review
https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before
opening a pull request. -->

<!-- Please add a reviewer to the assignee section when you create a PR.
If you don't have the access to it, we will shortly find a reviewer and
assign them to your PR. -->

## Why are these changes needed?

Since we only support pyarrow>=8.0.0, this condition check can be
removed.
@zhaoch23 
## Related issue number

<!-- For example: "Closes #1234" -->

## Checks

- [x] I've signed off every commit(by using the -s flag, i.e., `git
commit -s`) in this PR.
- [x] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for
https://docs.ray.io/en/master/.
- [ ] I've added any new APIs to the API Reference. For example, if I
added a
method in Tune, I've added it in `doc/source/tune/api/` under the
           corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a
few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
   - [x] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested :(

---------

Signed-off-by: zhilong <zhilong.chen@mail.mcgill.ca>
Co-authored-by: Zhaoch <c233zhao@uwaterloo.ca>
Co-authored-by: Praveen <praveeng@anyscale.com>

Author: 3 people

python/ray/air/util/tensor_extensions/arrow.py

@@ -27,9 +27,6 @@
 from ray.util.annotations import DeveloperAPI, PublicAPI
 
 PYARROW_VERSION = get_pyarrow_version()
-# Minimum version of Arrow that supports ExtensionScalars.
-# TODO(Clark): Remove conditional definition once we only support Arrow 8.0.0+.
-MIN_PYARROW_VERSION_SCALAR = parse_version("8.0.0")
 # Minimum version of Arrow that supports subclassable ExtensionScalars.
 # TODO(Clark): Remove conditional definition once we only support Arrow 9.0.0+.
 MIN_PYARROW_VERSION_SCALAR_SUBCLASS = parse_version("9.0.0")
@@ -59,17 +56,6 @@ def __init__(self, data_str: str):
         super().__init__(message)
 
 
-def _arrow_supports_extension_scalars():
-    """
-    Whether Arrow ExtensionScalars are supported in the current pyarrow version.
-
-    This returns True if the pyarrow version is 8.0.0+, or if the pyarrow version is
-    unknown.
-    """
-    # TODO(Clark): Remove utility once we only support Arrow 8.0.0+.
-    return PYARROW_VERSION is None or PYARROW_VERSION >= MIN_PYARROW_VERSION_SCALAR
-
-
 def _arrow_extension_scalars_are_subclassable():
     """
     Whether Arrow ExtensionScalars support subclassing in the current pyarrow version.
@@ -489,20 +475,16 @@ def __arrow_ext_scalar_class__(self):
             """
             return ArrowTensorScalar
 
-    if _arrow_supports_extension_scalars():
-        # TODO(Clark): Remove this version guard once we only support Arrow 8.0.0+.
-        def _extension_scalar_to_ndarray(
-            self, scalar: pa.ExtensionScalar
-        ) -> np.ndarray:
-            """
-            Convert an ExtensionScalar to a tensor element.
-            """
-            raw_values = scalar.value.values
-            shape = scalar.type.shape
-            value_type = raw_values.type
-            offset = raw_values.offset
-            data_buffer = raw_values.buffers()[1]
-            return _to_ndarray_helper(shape, value_type, offset, data_buffer)
+    def _extension_scalar_to_ndarray(self, scalar: "pa.ExtensionScalar") -> np.ndarray:
+        """
+        Convert an ExtensionScalar to a tensor element.
+        """
+        raw_values = scalar.value.values
+        shape = scalar.type.shape
+        value_type = raw_values.type
+        offset = raw_values.offset
+        data_buffer = raw_values.buffers()[1]
+        return _to_ndarray_helper(shape, value_type, offset, data_buffer)
 
     def __str__(self) -> str:
         return (
@@ -657,42 +639,20 @@ def to_pylist(self):
             # support (see comment in __getitem__).
             return list(self)
 
-        if _arrow_supports_extension_scalars():
-            # NOTE(Clark): This __getitem__ override is only needed for Arrow 8.*,
-            # before ExtensionScalar subclassing support was added.
-            # TODO(Clark): Remove these methods once we only support Arrow 9.0.0+.
-            def __getitem__(self, key):
-                # This __getitem__ hook allows us to support proper indexing when
-                # accessing a single tensor (a "scalar" item of the array). Without this
-                # hook for integer keys, the indexing will fail on pyarrow < 9.0.0 due
-                # to a lack of ExtensionScalar subclassing support.
-
-                # NOTE(Clark): We'd like to override the pa.Array.getitem() helper
-                # instead, which would obviate the need for overriding __iter__(), but
-                # unfortunately overriding Cython cdef methods with normal Python
-                # methods isn't allowed.
-                item = super().__getitem__(key)
-                if not isinstance(key, slice):
-                    item = item.type._extension_scalar_to_ndarray(item)
-                return item
+        def __getitem__(self, key):
+            # This __getitem__ hook allows us to support proper indexing when
+            # accessing a single tensor (a "scalar" item of the array). Without this
+            # hook for integer keys, the indexing will fail on pyarrow < 9.0.0 due
+            # to a lack of ExtensionScalar subclassing support.
 
-        else:
-            # NOTE(Clark): This __getitem__ override is only needed for Arrow < 8.0.0,
-            # before any ExtensionScalar support was added.
-            # TODO(Clark): Remove these methods once we only support Arrow 8.0.0+.
-            def __getitem__(self, key):
-                # This __getitem__ hook allows us to support proper indexing when
-                # accessing a single tensor (a "scalar" item of the array). Without this
-                # hook for integer keys, the indexing will fail on pyarrow < 8.0.0 due
-                # to a lack of ExtensionScalar support.
-
-                # NOTE(Clark): We'd like to override the pa.Array.getitem() helper
-                # instead, which would obviate the need for overriding __iter__(), but
-                # unfortunately overriding Cython cdef methods with normal Python
-                # methods isn't allowed.
-                if isinstance(key, slice):
-                    return super().__getitem__(key)
-                return self._to_numpy(key)
+            # NOTE(Clark): We'd like to override the pa.Array.getitem() helper
+            # instead, which would obviate the need for overriding __iter__(), but
+            # unfortunately overriding Cython cdef methods with normal Python
+            # methods isn't allowed.
+            item = super().__getitem__(key)
+            if not isinstance(key, slice):
+                item = item.type._extension_scalar_to_ndarray(item)
+            return item
 
 
 # NOTE: We need to inherit from the mixin before pa.ExtensionArray to ensure that the
@@ -1109,22 +1069,18 @@ def __str__(self) -> str:
     def __repr__(self) -> str:
         return str(self)
 
-    if _arrow_supports_extension_scalars():
-        # TODO(Clark): Remove this version guard once we only support Arrow 8.0.0+.
-        def _extension_scalar_to_ndarray(
-            self, scalar: pa.ExtensionScalar
-        ) -> np.ndarray:
-            """
-            Convert an ExtensionScalar to a tensor element.
-            """
-            data = scalar.value.get("data")
-            raw_values = data.values
-
-            shape = tuple(scalar.value.get("shape").as_py())
-            value_type = raw_values.type
-            offset = raw_values.offset
-            data_buffer = raw_values.buffers()[1]
-            return _to_ndarray_helper(shape, value_type, offset, data_buffer)
+    def _extension_scalar_to_ndarray(self, scalar: "pa.ExtensionScalar") -> np.ndarray:
+        """
+        Convert an ExtensionScalar to a tensor element.
+        """
+        data = scalar.value.get("data")
+        raw_values = data.values
+
+        shape = tuple(scalar.value.get("shape").as_py())
+        value_type = raw_values.type
+        offset = raw_values.offset
+        data_buffer = raw_values.buffers()[1]
+        return _to_ndarray_helper(shape, value_type, offset, data_buffer)
 
 
 # NOTE: We need to inherit from the mixin before pa.ExtensionArray to ensure that the