update

Author: tchaton

src/lightning/data/processing/data_processor.py

@@ -29,13 +29,13 @@
     _TORCH_GREATER_EQUAL_2_1_0,
 )
 from lightning.data.processing.readers import BaseReader
+from lightning.data.processing.utilities import _create_dataset
 from lightning.data.streaming import Cache
 from lightning.data.streaming.cache import Dir
 from lightning.data.streaming.client import S3Client
 from lightning.data.streaming.resolver import _resolve_dir
 from lightning.data.utilities.broadcast import broadcast_object
 from lightning.data.utilities.packing import _pack_greedily
-from lightning.data.processing.utilities import _create_dataset
 
 if _TORCH_GREATER_EQUAL_2_1_0:
     from torch.utils._pytree import tree_flatten, tree_unflatten, treespec_loads

src/lightning/data/processing/utilities.py

@@ -3,20 +3,17 @@
 import urllib
 from contextlib import contextmanager
 from subprocess import Popen
-from typing import Any, Callable, Optional, Tuple
+from typing import Any, Callable, List, Optional, Tuple, Union
 
 from lightning.data.constants import _IS_IN_STUDIO, _LIGHTNING_CLOUD_LATEST
 
-from typing import Optional, List, Tuple, Union
-import os
-
 if _LIGHTNING_CLOUD_LATEST:
-    from lightning_cloud.rest_client import LightningClient
     from lightning_cloud.openapi import (
         ProjectIdDatasetsBody,
         V1DatasetType,
     )
     from lightning_cloud.openapi.rest import ApiException
+    from lightning_cloud.rest_client import LightningClient
 
 
 def _create_dataset(
@@ -33,9 +30,7 @@ def _create_dataset(
     name: Optional[str] = None,
     version: Optional[int] = None,
 ):
-    """
-    Create a dataset with metadata information about its source and destination
-    """
+    """Create a dataset with metadata information about its source and destination."""
     project_id = os.getenv("LIGHTNING_CLOUD_PROJECT_ID", None)
     cluster_id = os.getenv("LIGHTNING_CLUSTER_ID", None)
     user_id = os.getenv("LIGHTNING_USER_ID", None)
@@ -70,10 +65,10 @@ def _create_dataset(
                 type=dataset_type,
                 version=version,
             ),
-            project_id=project_id
+            project_id=project_id,
         )
     except ApiException as ex:
-        if 'already exists' in str(ex.body):
+        if "already exists" in str(ex.body):
             pass
         else:
             raise ex