update

Author: thomas

src/lightning/data/README.md

@@ -311,7 +311,7 @@ class ImagenetStreamingDataset(StreamingDataset):
 
     def __getitem__(self, index):
         image = super().__getitem__(index)
-        return T.resize(image, (224, 224))
+        return F.resize(image, (224, 224))
 
 dataset = ImagenetStreamingDataset(...)
 dataloader = StreamingDataLoader(dataset, batch_size=4)

src/lightning/data/processing/functions.py

@@ -157,6 +157,7 @@ def map(
     num_nodes: Optional[int] = None,
     machine: Optional[str] = None,
     num_downloaders: Optional[int] = None,
+    num_uploaders: Optional[int] = None,
     reorder_files: bool = True,
     error_when_not_empty: bool = False,
     reader: Optional[BaseReader] = None,
@@ -173,6 +174,7 @@ def map(
         num_nodes: When doing remote execution, the number of nodes to use. Only supported on https://lightning.ai/.
         machine: When doing remote execution, the machine to use. Only supported on https://lightning.ai/.
         num_downloaders: The number of downloaders per worker.
+        num_uploaders: The number of uploaders per workers.
         reorder_files: By default, reorders the files by file size to distribute work equally among all workers.
             Set this to ``False`` if the order in which samples are processed should be preserved.
         error_when_not_empty: Whether we should error if the output folder isn't empty.
@@ -215,6 +217,7 @@ def map(
             num_workers=num_workers or os.cpu_count(),
             fast_dev_run=fast_dev_run,
             num_downloaders=num_downloaders,
+            num_uploaders=num_uploaders,
             reorder_files=reorder_files,
             weights=weights,
             reader=reader,
@@ -240,6 +243,7 @@ def optimize(
     num_nodes: Optional[int] = None,
     machine: Optional[str] = None,
     num_downloaders: Optional[int] = None,
+    num_uploaders: Optional[int] = None,
     reorder_files: bool = True,
     reader: Optional[BaseReader] = None,
 ) -> None:
@@ -258,6 +262,7 @@ def optimize(
         num_nodes: When doing remote execution, the number of nodes to use. Only supported on https://lightning.ai/.
         machine: When doing remote execution, the machine to use. Only supported on https://lightning.ai/.
         num_downloaders: The number of downloaders per worker.
+        num_uploaders: The numbers of uploaders per worker.
         reorder_files: By default, reorders the files by file size to distribute work equally among all workers.
             Set this to ``False`` if the order in which samples are processed should be preserved.
 
@@ -302,6 +307,7 @@ def optimize(
             num_workers=num_workers or os.cpu_count(),
             fast_dev_run=fast_dev_run,
             num_downloaders=num_downloaders,
+            num_uploaders=num_uploaders,
             reorder_files=reorder_files,
             reader=reader,
         )