Remove ray lazy loader.

data_juicer/core/ray_data.py

@@ -5,19 +5,17 @@
 from typing import Any, Dict, List, Literal, Optional, Union
 
 import pyarrow
+import ray.data as rd
+import ray.data.read_api as ds
 from loguru import logger
 
 from data_juicer import cuda_device_count
 from data_juicer.core.data import DJDataset
 from data_juicer.ops import Deduplicator, Filter, Mapper
 from data_juicer.ops.base_op import TAGGING_OPS
 from data_juicer.utils.constant import Fields
-from data_juicer.utils.lazy_loader import LazyLoader
 from data_juicer.utils.process_utils import calculate_np
 
-rd = LazyLoader('rd', 'ray.data')
-ds = LazyLoader('ds', 'ray.data.read_api')
-
 
 def get_abs_path(path, dataset_dir):
     full_path = os.path.abspath(os.path.join(dataset_dir, path))

data_juicer/core/ray_executor.py

@@ -2,19 +2,16 @@
 import shutil
 import time
 
+import ray
 from loguru import logger
 
 from data_juicer.config import init_configs
 from data_juicer.core.ray_data import RayDataset
 from data_juicer.ops import load_ops
 from data_juicer.ops.op_fusion import fuse_operators
-from data_juicer.utils.lazy_loader import LazyLoader
 
 from .adapter import Adapter
 
-ray = LazyLoader('ray', 'ray')
-rd = LazyLoader('rd', 'ray.data')
-
 
 class TempDirManager:
 

data_juicer/format/empty_formatter.py

@@ -1,14 +1,11 @@
 from typing import List
 
 import pandas as pd
+import ray
 from datasets import Dataset, Features, Value
 
-from data_juicer.utils.lazy_loader import LazyLoader
-
 from .formatter import FORMATTERS, BaseFormatter
 
-ray = LazyLoader('ray', 'ray')
-
 
 @FORMATTERS.register_module()
 class EmptyFormatter(BaseFormatter):

data_juicer/utils/unittest_utils.py

@@ -3,14 +3,14 @@
 import unittest
 
 import numpy
+import ray.data as rd
 
 from data_juicer import is_cuda_available
 from data_juicer.core.data import DJDataset, NestedDataset
 from data_juicer.core.ray_data import RayDataset
 from data_juicer.utils.lazy_loader import LazyLoader
 from data_juicer.utils.model_utils import free_models
 
-rd = LazyLoader('rd', 'ray.data')
 transformers = LazyLoader('transformers', 'transformers')
 
 CLEAR_MODEL = False