From ce2d587bc40a29ab568aaed238319fdc037aca12 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Wed, 7 Feb 2024 23:14:45 +0000 Subject: [PATCH 1/8] update --- _notebooks | 1 - src/lightning/data/processing/__init__.py | 26 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) delete mode 160000 _notebooks diff --git a/_notebooks b/_notebooks deleted file mode 160000 index 543a8d8200662..0000000000000 --- a/_notebooks +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 543a8d82006620906dc9eb669eab18d06ebe6863 diff --git a/src/lightning/data/processing/__init__.py b/src/lightning/data/processing/__init__.py index e69de29bb2d1d..7ffdf9cf23c4d 100644 --- a/src/lightning/data/processing/__init__.py +++ b/src/lightning/data/processing/__init__.py @@ -0,0 +1,26 @@ +from subprocess import Popen + +def optimize_dns(enable): + with open("/etc/resolv.conf", "r") as f: + lines = f.readlines() + + if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): + Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() + +def _optimize_dns(enable=False): + with open("/etc/resolv.conf", "r") as f: + lines = f.readlines() + + write_lines = [] + for line in lines: + if "nameserver 127" in line: + if enable: + write_lines.append('nameserver 127.0.0.1\n') + else: + write_lines.append('nameserver 127.0.0.53\n') + else: + write_lines.append(line) + + with open("/etc/resolv.conf", "w") as f: + for line in write_lines: + f.write(line) \ No newline at end of file From ecf5b69fcd8588c213a0f6d45a3534172a7dceae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:17:31 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning/data/processing/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lightning/data/processing/__init__.py b/src/lightning/data/processing/__init__.py index 7ffdf9cf23c4d..6bf715cbd80d2 100644 --- a/src/lightning/data/processing/__init__.py +++ b/src/lightning/data/processing/__init__.py @@ -1,14 +1,15 @@ from subprocess import Popen + def optimize_dns(enable): - with open("/etc/resolv.conf", "r") as f: + with open("/etc/resolv.conf") as f: lines = f.readlines() if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() def _optimize_dns(enable=False): - with open("/etc/resolv.conf", "r") as f: + with open("/etc/resolv.conf") as f: lines = f.readlines() write_lines = [] @@ -23,4 +24,4 @@ def _optimize_dns(enable=False): with open("/etc/resolv.conf", "w") as f: for line in write_lines: - f.write(line) \ No newline at end of file + f.write(line) From 21caacdccc346e22c432b161c1868d85446e6b4f Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 7 Feb 2024 23:48:28 +0000 Subject: [PATCH 3/8] update --- _notebooks | 1 + .../data/{streaming => }/constants.py | 0 src/lightning/data/processing/__init__.py | 27 ------------ src/lightning/data/processing/dns.py | 42 +++++++++++++++++++ src/lightning/data/streaming/cache.py | 2 +- src/lightning/data/streaming/client.py | 2 +- src/lightning/data/streaming/config.py | 2 +- .../data/streaming/data_processor.py | 10 ++--- src/lightning/data/streaming/dataloader.py | 2 +- src/lightning/data/streaming/dataset.py | 4 +- src/lightning/data/streaming/downloader.py | 2 +- src/lightning/data/streaming/functions.py | 25 ++++++----- src/lightning/data/streaming/item_loader.py | 2 +- src/lightning/data/streaming/reader.py | 2 +- src/lightning/data/streaming/serializers.py | 2 +- src/lightning/data/streaming/writer.py | 2 +- 16 files changed, 74 insertions(+), 53 deletions(-) create mode 160000 _notebooks rename src/lightning/data/{streaming => }/constants.py (100%) create mode 100644 src/lightning/data/processing/dns.py diff --git a/_notebooks b/_notebooks new file mode 160000 index 0000000000000..543a8d8200662 --- /dev/null +++ b/_notebooks @@ -0,0 +1 @@ +Subproject commit 543a8d82006620906dc9eb669eab18d06ebe6863 diff --git a/src/lightning/data/streaming/constants.py b/src/lightning/data/constants.py similarity index 100% rename from src/lightning/data/streaming/constants.py rename to src/lightning/data/constants.py diff --git a/src/lightning/data/processing/__init__.py b/src/lightning/data/processing/__init__.py index 6bf715cbd80d2..e69de29bb2d1d 100644 --- a/src/lightning/data/processing/__init__.py +++ b/src/lightning/data/processing/__init__.py @@ -1,27 +0,0 @@ -from subprocess import Popen - - -def optimize_dns(enable): - with open("/etc/resolv.conf") as f: - lines = f.readlines() - - if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): - Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() - -def _optimize_dns(enable=False): - with open("/etc/resolv.conf") as f: - lines = f.readlines() - - write_lines = [] - for line in lines: - if "nameserver 127" in line: - if enable: - write_lines.append('nameserver 127.0.0.1\n') - else: - write_lines.append('nameserver 127.0.0.53\n') - else: - write_lines.append(line) - - with open("/etc/resolv.conf", "w") as f: - for line in write_lines: - f.write(line) diff --git a/src/lightning/data/processing/dns.py b/src/lightning/data/processing/dns.py new file mode 100644 index 0000000000000..114ef1720c681 --- /dev/null +++ b/src/lightning/data/processing/dns.py @@ -0,0 +1,42 @@ +from contextlib import contextmanager +from subprocess import Popen + +from lightning.data.constants import _IS_IN_STUDIO + + +@contextmanager +def optimize_dns_context(enable): + optimize_dns(enable) + try: + yield + except Exception as e: + optimize_dns(not enable) + raise e + +def optimize_dns(enable): + if not _IS_IN_STUDIO: + return + + with open("/etc/resolv.conf") as f: + lines = f.readlines() + + if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): # noqa E501 + Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 + +def _optimize_dns(enable=False): + with open("/etc/resolv.conf") as f: + lines = f.readlines() + + write_lines = [] + for line in lines: + if "nameserver 127" in line: + if enable: + write_lines.append('nameserver 127.0.0.1\n') + else: + write_lines.append('nameserver 127.0.0.53\n') + else: + write_lines.append(line) + + with open("/etc/resolv.conf", "w") as f: + for line in write_lines: + f.write(line) diff --git a/src/lightning/data/streaming/cache.py b/src/lightning/data/streaming/cache.py index 667409c81b8d8..305f393d276ac 100644 --- a/src/lightning/data/streaming/cache.py +++ b/src/lightning/data/streaming/cache.py @@ -15,7 +15,7 @@ import os from typing import Any, Dict, List, Optional, Tuple, Union -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _INDEX_FILENAME, _LIGHTNING_CLOUD_LATEST, _TORCH_GREATER_EQUAL_2_1_0, diff --git a/src/lightning/data/streaming/client.py b/src/lightning/data/streaming/client.py index c42d3af9f6ed1..4a572548854de 100644 --- a/src/lightning/data/streaming/client.py +++ b/src/lightning/data/streaming/client.py @@ -2,7 +2,7 @@ from time import time from typing import Any, Optional -from lightning.data.streaming.constants import _BOTO3_AVAILABLE +from lightning.data.constants import _BOTO3_AVAILABLE if _BOTO3_AVAILABLE: import boto3 diff --git a/src/lightning/data/streaming/config.py b/src/lightning/data/streaming/config.py index 386ea475e7681..4a5a4ba8c55b2 100644 --- a/src/lightning/data/streaming/config.py +++ b/src/lightning/data/streaming/config.py @@ -15,7 +15,7 @@ import os from typing import Any, Dict, List, Optional, Tuple -from lightning.data.streaming.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 +from lightning.data.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.downloader import get_downloader_cls from lightning.data.streaming.item_loader import BaseItemLoader, PyTreeLoader, TokensLoader from lightning.data.streaming.sampler import ChunkedIndex diff --git a/src/lightning/data/streaming/data_processor.py b/src/lightning/data/streaming/data_processor.py index 0449a1218623a..3a757857b08f8 100644 --- a/src/lightning/data/streaming/data_processor.py +++ b/src/lightning/data/streaming/data_processor.py @@ -20,11 +20,7 @@ from tqdm.auto import tqdm as _tqdm from lightning import seed_everything -from lightning.data.processing.readers import BaseReader -from lightning.data.streaming import Cache -from lightning.data.streaming.cache import Dir -from lightning.data.streaming.client import S3Client -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _BOTO3_AVAILABLE, _DEFAULT_FAST_DEV_RUN_ITEMS, _INDEX_FILENAME, @@ -32,6 +28,10 @@ _LIGHTNING_CLOUD_LATEST, _TORCH_GREATER_EQUAL_2_1_0, ) +from lightning.data.processing.readers import BaseReader +from lightning.data.streaming import Cache +from lightning.data.streaming.cache import Dir +from lightning.data.streaming.client import S3Client from lightning.data.streaming.resolver import _resolve_dir from lightning.data.utilities.broadcast import broadcast_object from lightning.data.utilities.packing import _pack_greedily diff --git a/src/lightning/data/streaming/dataloader.py b/src/lightning/data/streaming/dataloader.py index 942d2f98a3090..04793296ca30f 100644 --- a/src/lightning/data/streaming/dataloader.py +++ b/src/lightning/data/streaming/dataloader.py @@ -33,13 +33,13 @@ ) from torch.utils.data.sampler import BatchSampler, Sampler +from lightning.data.constants import _DEFAULT_CHUNK_BYTES, _TORCH_GREATER_EQUAL_2_1_0, _VIZ_TRACKER_AVAILABLE from lightning.data.streaming import Cache from lightning.data.streaming.combined import ( __NUM_SAMPLES_YIELDED_KEY__, __SAMPLES_KEY__, CombinedStreamingDataset, ) -from lightning.data.streaming.constants import _DEFAULT_CHUNK_BYTES, _TORCH_GREATER_EQUAL_2_1_0, _VIZ_TRACKER_AVAILABLE from lightning.data.streaming.dataset import StreamingDataset from lightning.data.streaming.sampler import CacheBatchSampler from lightning.data.utilities.env import _DistributedEnv diff --git a/src/lightning/data/streaming/dataset.py b/src/lightning/data/streaming/dataset.py index e281bbc0a1e2d..da0028184e123 100644 --- a/src/lightning/data/streaming/dataset.py +++ b/src/lightning/data/streaming/dataset.py @@ -19,11 +19,11 @@ import numpy as np from torch.utils.data import IterableDataset -from lightning.data.streaming import Cache -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _DEFAULT_CACHE_DIR, _INDEX_FILENAME, ) +from lightning.data.streaming import Cache from lightning.data.streaming.item_loader import BaseItemLoader from lightning.data.streaming.resolver import Dir, _resolve_dir from lightning.data.streaming.sampler import ChunkedIndex diff --git a/src/lightning/data/streaming/downloader.py b/src/lightning/data/streaming/downloader.py index 982b4d25142b3..03d7b9302068a 100644 --- a/src/lightning/data/streaming/downloader.py +++ b/src/lightning/data/streaming/downloader.py @@ -19,8 +19,8 @@ from filelock import FileLock, Timeout +from lightning.data.constants import _INDEX_FILENAME from lightning.data.streaming.client import S3Client -from lightning.data.streaming.constants import _INDEX_FILENAME class Downloader(ABC): diff --git a/src/lightning/data/streaming/functions.py b/src/lightning/data/streaming/functions.py index 7bc865ccb8d85..3cf87f7699f1d 100644 --- a/src/lightning/data/streaming/functions.py +++ b/src/lightning/data/streaming/functions.py @@ -22,8 +22,9 @@ import torch +from lightning.data.constants import _IS_IN_STUDIO, _TORCH_GREATER_EQUAL_2_1_0 +from lightning.data.processing.dns import optimize_dns_context from lightning.data.processing.readers import BaseReader -from lightning.data.streaming.constants import _IS_IN_STUDIO, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.data_processor import DataChunkRecipe, DataProcessor, DataTransformRecipe from lightning.data.streaming.resolver import ( Dir, @@ -218,7 +219,8 @@ def map( weights=weights, reader=reader, ) - return data_processor.run(LambdaDataTransformRecipe(fn, inputs)) + with optimize_dns_context(True): + return data_processor.run(LambdaDataTransformRecipe(fn, inputs)) return _execute( f"data-prep-map-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", num_nodes, @@ -303,15 +305,18 @@ def optimize( reorder_files=reorder_files, reader=reader, ) - return data_processor.run( - LambdaDataChunkRecipe( - fn, - inputs, - chunk_size=chunk_size, - chunk_bytes=chunk_bytes, - compression=compression, + + with optimize_dns_context(True): + data_processor.run( + LambdaDataChunkRecipe( + fn, + inputs, + chunk_size=chunk_size, + chunk_bytes=chunk_bytes, + compression=compression, + ) ) - ) + return None return _execute( f"data-prep-optimize-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}", num_nodes, diff --git a/src/lightning/data/streaming/item_loader.py b/src/lightning/data/streaming/item_loader.py index 779a683146182..2a1a02da67293 100644 --- a/src/lightning/data/streaming/item_loader.py +++ b/src/lightning/data/streaming/item_loader.py @@ -19,7 +19,7 @@ import numpy as np import torch -from lightning.data.streaming.constants import ( +from lightning.data.constants import ( _TORCH_DTYPES_MAPPING, _TORCH_GREATER_EQUAL_2_1_0, ) diff --git a/src/lightning/data/streaming/reader.py b/src/lightning/data/streaming/reader.py index 50705cb18f663..298452ea685f7 100644 --- a/src/lightning/data/streaming/reader.py +++ b/src/lightning/data/streaming/reader.py @@ -20,8 +20,8 @@ from threading import Thread from typing import Any, Dict, List, Optional, Tuple, Union +from lightning.data.constants import _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.config import ChunksConfig -from lightning.data.streaming.constants import _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.item_loader import BaseItemLoader, PyTreeLoader from lightning.data.streaming.sampler import ChunkedIndex from lightning.data.streaming.serializers import Serializer, _get_serializers diff --git a/src/lightning/data/streaming/serializers.py b/src/lightning/data/streaming/serializers.py index b689429953c21..9e5a92f520741 100644 --- a/src/lightning/data/streaming/serializers.py +++ b/src/lightning/data/streaming/serializers.py @@ -23,7 +23,7 @@ import torch from lightning_utilities.core.imports import RequirementCache -from lightning.data.streaming.constants import _NUMPY_DTYPES_MAPPING, _TORCH_DTYPES_MAPPING +from lightning.data.constants import _NUMPY_DTYPES_MAPPING, _TORCH_DTYPES_MAPPING _PIL_AVAILABLE = RequirementCache("PIL") _TORCH_VISION_AVAILABLE = RequirementCache("torchvision") diff --git a/src/lightning/data/streaming/writer.py b/src/lightning/data/streaming/writer.py index 44e6a8951773f..98b7a31e07d1b 100644 --- a/src/lightning/data/streaming/writer.py +++ b/src/lightning/data/streaming/writer.py @@ -21,8 +21,8 @@ import numpy as np import torch +from lightning.data.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.compression import _COMPRESSORS, Compressor -from lightning.data.streaming.constants import _INDEX_FILENAME, _TORCH_GREATER_EQUAL_2_1_0 from lightning.data.streaming.serializers import Serializer, _get_serializers from lightning.data.utilities.env import _DistributedEnv, _WorkerEnv from lightning.data.utilities.format import _convert_bytes_to_int, _human_readable_bytes From 08b5767536fff6a89ef5e68dd566c0b13b457f52 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 8 Feb 2024 09:02:22 +0000 Subject: [PATCH 4/8] update --- src/lightning/data/processing/dns.py | 7 ++++--- tests/tests_data/processing/test_dns.py | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/tests_data/processing/test_dns.py diff --git a/src/lightning/data/processing/dns.py b/src/lightning/data/processing/dns.py index a28cf4eb41f4e..f4940d118f17e 100644 --- a/src/lightning/data/processing/dns.py +++ b/src/lightning/data/processing/dns.py @@ -1,11 +1,12 @@ from contextlib import contextmanager from subprocess import Popen +from typing import Any from lightning.data.constants import _IS_IN_STUDIO @contextmanager -def optimize_dns_context(enable): +def optimize_dns_context(enable: bool) -> Any: optimize_dns(enable) try: yield @@ -14,7 +15,7 @@ def optimize_dns_context(enable): optimize_dns(False) # always disable the optimize DNS raise e -def optimize_dns(enable): +def optimize_dns(enable: bool) -> None: if not _IS_IN_STUDIO: return @@ -24,7 +25,7 @@ def optimize_dns(enable): if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): # noqa E501 Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 -def _optimize_dns(enable=False): +def _optimize_dns(enable: bool) -> None: with open("/etc/resolv.conf") as f: lines = f.readlines() diff --git a/tests/tests_data/processing/test_dns.py b/tests/tests_data/processing/test_dns.py new file mode 100644 index 0000000000000..2efa23d22ea0d --- /dev/null +++ b/tests/tests_data/processing/test_dns.py @@ -0,0 +1,7 @@ +from lightning.data.processing.dns import optimize_dns_context + + +def test_optimize_dns_context(monkeypatch): + + with optimize_dns_context(True): + pass From 1a0dfcab4a9e23df4592645c627dfabbb596ecac Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 8 Feb 2024 09:12:18 +0000 Subject: [PATCH 5/8] update --- resolv.conf | 17 ++++++++++++++++ src/lightning/data/processing/dns.py | 5 ++++- tests/tests_data/processing/test_dns.py | 26 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 resolv.conf diff --git a/resolv.conf b/resolv.conf new file mode 100644 index 0000000000000..efd5a82229904 --- /dev/null +++ b/resolv.conf @@ -0,0 +1,17 @@ +# +# macOS Notice +# +# This file is not consulted for DNS hostname resolution, address +# resolution, or the DNS query routing mechanism used by most +# processes on this system. +# +# To view the DNS configuration used by this system, use: +# scutil --dns +# +# SEE ALSO +# dns-sd(1), scutil(8) +# +# This file is automatically generated. +# +search tailead5b.ts.net Home +nameserver 100.100.100.100 diff --git a/src/lightning/data/processing/dns.py b/src/lightning/data/processing/dns.py index f4940d118f17e..bb058dd3a799a 100644 --- a/src/lightning/data/processing/dns.py +++ b/src/lightning/data/processing/dns.py @@ -22,7 +22,10 @@ def optimize_dns(enable: bool) -> None: with open("/etc/resolv.conf") as f: lines = f.readlines() - if (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)): # noqa E501 + if ( + (enable and any("127.0.0.53" in line for line in lines)) + or (not enable and any("127.0.0.1" in line for line in lines)) + ): # noqa E501 Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 def _optimize_dns(enable: bool) -> None: diff --git a/tests/tests_data/processing/test_dns.py b/tests/tests_data/processing/test_dns.py index 2efa23d22ea0d..40cd9bc199161 100644 --- a/tests/tests_data/processing/test_dns.py +++ b/tests/tests_data/processing/test_dns.py @@ -1,7 +1,33 @@ +from unittest.mock import MagicMock + +from lightning.data.processing import dns as dns_module from lightning.data.processing.dns import optimize_dns_context def test_optimize_dns_context(monkeypatch): + popen_mock = MagicMock() + + monkeypatch.setattr(dns_module, "_IS_IN_STUDIO", True) + monkeypatch.setattr(dns_module, "Popen", popen_mock) + + class FakeFile: + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + return self + + def readlines(self): + return ["127.0.0.53"] + + monkeypatch.setitem(__builtins__, "open", MagicMock(return_value=FakeFile())) with optimize_dns_context(True): pass + + cmd = popen_mock._mock_call_args_list[0].args[0] + assert cmd == "sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns(True)'" # noqa: E501 From a10a55e4b0a297be470045cb8479f9af96e0738e Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 8 Feb 2024 09:12:28 +0000 Subject: [PATCH 6/8] update --- resolv.conf | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 resolv.conf diff --git a/resolv.conf b/resolv.conf deleted file mode 100644 index efd5a82229904..0000000000000 --- a/resolv.conf +++ /dev/null @@ -1,17 +0,0 @@ -# -# macOS Notice -# -# This file is not consulted for DNS hostname resolution, address -# resolution, or the DNS query routing mechanism used by most -# processes on this system. -# -# To view the DNS configuration used by this system, use: -# scutil --dns -# -# SEE ALSO -# dns-sd(1), scutil(8) -# -# This file is automatically generated. -# -search tailead5b.ts.net Home -nameserver 100.100.100.100 From 90df94baeb64095acc2c398e901db72de664b8ce Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 8 Feb 2024 09:24:50 +0000 Subject: [PATCH 7/8] update --- src/lightning/data/processing/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning/data/processing/dns.py b/src/lightning/data/processing/dns.py index bb058dd3a799a..f1ca83dbc0e2e 100644 --- a/src/lightning/data/processing/dns.py +++ b/src/lightning/data/processing/dns.py @@ -26,7 +26,7 @@ def optimize_dns(enable: bool) -> None: (enable and any("127.0.0.53" in line for line in lines)) or (not enable and any("127.0.0.1" in line for line in lines)) ): # noqa E501 - Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 + Popen(f"sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing.dns import _optimize_dns; _optimize_dns({enable})'", shell=True).wait() # noqa E501 def _optimize_dns(enable: bool) -> None: with open("/etc/resolv.conf") as f: From f857e25ef1d531789f72517d7bb7e8365636a3e5 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 8 Feb 2024 09:25:12 +0000 Subject: [PATCH 8/8] update --- tests/tests_data/processing/test_dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_data/processing/test_dns.py b/tests/tests_data/processing/test_dns.py index 40cd9bc199161..0703e346b02fd 100644 --- a/tests/tests_data/processing/test_dns.py +++ b/tests/tests_data/processing/test_dns.py @@ -30,4 +30,4 @@ def readlines(self): pass cmd = popen_mock._mock_call_args_list[0].args[0] - assert cmd == "sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing import _optimize_dns; _optimize_dns(True)'" # noqa: E501 + assert cmd == "sudo /home/zeus/miniconda3/envs/cloudspace/bin/python -c 'from lightning.data.processing.dns import _optimize_dns; _optimize_dns(True)'" # noqa: E501