Streaming Dataset: tiny optimisations (#19342)

tchaton · web-flow · commit 925357d2e9ac · 2024-01-24T19:43:33.000Z
diff --git a/src/lightning/data/streaming/downloader.py b/src/lightning/data/streaming/downloader.py
@@ -59,7 +59,7 @@ def download_file(self, remote_filepath: str, local_filepath: str) -> None:
             with FileLock(local_filepath + ".lock", timeout=0):
                 if self._s5cmd_available:
                     proc = subprocess.Popen(
-                        f"s5cmd --numworkers 64 cp {remote_filepath} {local_filepath}",
+                        f"s5cmd cp {remote_filepath} {local_filepath}",
                         shell=True,
                         stdout=subprocess.PIPE,
                     )

Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ def download_file(self, remote_filepath: str, local_filepath: str) -> None:`
`59`	`59`	`with FileLock(local_filepath + ".lock", timeout=0):`
`60`	`60`	`if self._s5cmd_available:`
`61`	`61`	`proc = subprocess.Popen(`
`62`		`- f"s5cmd --numworkers 64 cp {remote_filepath} {local_filepath}",`
	`62`	`+ f"s5cmd cp {remote_filepath} {local_filepath}",`
`63`	`63`	`shell=True,`
`64`	`64`	`stdout=subprocess.PIPE,`
`65`	`65`	`)`