lossened up the huggingface dependencies (#89)

Author: ChaoPang

pyproject.toml

@@ -28,7 +28,7 @@ classifiers = [
 dependencies = [
     "dask==2024.1.1",
     "dask[dataframe]==2024.1.1",
-    "datasets==2.16.1",
+    "datasets>=2.16.1",
     "evaluate==0.4.1",
     "fast-ml==3.68",
     "femr==0.2.0",
@@ -39,7 +39,7 @@ dependencies = [
     "numpy==1.24.3",
     "packaging==23.2",
     "pandas==2.2.0",
-    "peft==0.10.0",
+    "peft>=0.10.0",
     "Pillow==10.3.0",
     "pyarrow==15.0.0",
     "pydantic==2.6.0",
@@ -50,13 +50,13 @@ dependencies = [
     "tensorflow==2.15.0",
     "tensorflow-metal==1.1.0; sys_platform == 'darwin'", # macOS only
     "tensorflow-datasets==4.5.2",
-    "tqdm==4.66.1",
+    "tqdm>=4.66.1",
     "torch==2.4.0",
-    "tokenizers==0.19.0",
-    "transformers==4.40.0",
-    "accelerate==0.31.0",
+    "tokenizers>=0.19.0",
+    "transformers>=4.40.0",
+    "accelerate>=0.31.0",
     "Werkzeug==3.0.1",
-    "wandb==0.17.8",
+    "wandb>=0.17.8",
     "xgboost==2.0.3",
     "cehrbert_data==0.0.5"
 ]

src/cehrbert/runners/hf_cehrbert_finetune_runner.py

@@ -126,7 +126,7 @@ def main():
                 LOG.exception(e)
                 dataset = create_dataset_from_meds_reader(data_args, is_pretraining=False)
                 if not data_args.streaming:
-                    dataset.save_to_disk(meds_extension_path)
+                    dataset.save_to_disk(str(meds_extension_path))
             train_set = dataset["train"]
             validation_set = dataset["validation"]
             test_set = dataset["test"]
@@ -249,7 +249,7 @@ def assign_split(example):
         )
 
         if not data_args.streaming:
-            processed_dataset.save_to_disk(prepared_ds_path)
+            processed_dataset.save_to_disk(str(prepared_ds_path))
 
     collator = CehrBertDataCollator(tokenizer, model_args.max_position_embeddings, is_pretraining=False)
 

src/cehrbert/runners/hf_cehrbert_pretrain_runner.py

@@ -189,7 +189,7 @@ def main():
                 LOG.exception(e)
                 dataset = create_dataset_from_meds_reader(data_args, is_pretraining=True)
                 if not data_args.streaming:
-                    dataset.save_to_disk(meds_extension_path)
+                    dataset.save_to_disk(str(meds_extension_path))
         else:
             # Load the dataset from the parquet files
             dataset = load_parquet_as_dataset(
@@ -225,7 +225,7 @@ def main():
         )
         # only save the data to the disk if it is not streaming
         if not data_args.streaming:
-            processed_dataset.save_to_disk(prepared_ds_path)
+            processed_dataset.save_to_disk(str(prepared_ds_path))
 
     def filter_func(examples):
         return [_ >= data_args.min_num_tokens for _ in examples["num_of_concepts"]]