update

Author: tchaton

src/lightning/data/processing/data_processor.py

@@ -974,6 +974,7 @@ def run(self, data_recipe: DataRecipe) -> None:
         result = data_recipe._done(len(user_items), self.delete_cached_files, self.output_dir)
 
         if num_nodes == node_rank + 1 and self.output_dir.url and _IS_IN_STUDIO:
+            assert self.output_dir.path
             _create_dataset(
                 input_dir=self.input_dir.path,
                 storage_dir=self.output_dir.path,

src/lightning/data/processing/utilities.py

@@ -17,19 +17,19 @@
 
 
 def _create_dataset(
-    input_dir: str,
+    input_dir: Optional[str],
     storage_dir: str,
     dataset_type: V1DatasetType,
     empty: Optional[bool] = None,
     size: Optional[int] = None,
-    num_bytes: Optional[int] = None,
+    num_bytes: Optional[str] = None,
     data_format: Optional[Union[str, Tuple[str]]] = None,
     compression: Optional[str] = None,
-    num_chunks: Optional[str] = None,
+    num_chunks: Optional[int] = None,
     num_bytes_per_chunk: Optional[List[int]] = None,
     name: Optional[str] = None,
     version: Optional[int] = None,
-):
+) -> None:
     """Create a dataset with metadata information about its source and destination."""
     project_id = os.getenv("LIGHTNING_CLOUD_PROJECT_ID", None)
     cluster_id = os.getenv("LIGHTNING_CLUSTER_ID", None)