python wheels: do not "expand" (spack#43317)

Author: haampie

lib/spack/llnl/url.py

@@ -12,7 +12,7 @@
 # Archive extensions allowed in Spack
 PREFIX_EXTENSIONS = ("tar", "TAR")
 EXTENSIONS = ("gz", "bz2", "xz", "Z")
-NO_TAR_EXTENSIONS = ("zip", "tgz", "tbz2", "tbz", "txz")
+NO_TAR_EXTENSIONS = ("zip", "tgz", "tbz2", "tbz", "txz", "whl")
 
 # Add PREFIX_EXTENSIONS and EXTENSIONS last so that .tar.gz is matched *before* .tar or .gz
 ALLOWED_ARCHIVE_TYPES = (
@@ -403,7 +403,7 @@ def expand_contracted_extension_in_path(
 def compression_ext_from_compressed_archive(extension: str) -> Optional[str]:
     """Returns compression extension for a compressed archive"""
     extension = expand_contracted_extension(extension)
-    for ext in [*EXTENSIONS]:
+    for ext in EXTENSIONS:
         if ext in extension:
             return ext
     return None

lib/spack/spack/test/llnl/url.py

@@ -148,6 +148,8 @@ def test_strip_compression_extension(archive_and_expected):
         assert stripped == "Foo.zip"
         stripped = llnl.url.strip_compression_extension(archive, "zip")
         assert stripped == "Foo"
+    elif extension == "whl":
+        assert stripped == "Foo.whl"
     elif (
         extension.lower() == "tar"
         or extension in llnl.url.CONTRACTION_MAP

lib/spack/spack/test/util/compression.py

@@ -29,7 +29,9 @@
 ]
 # Spack does not use Python native handling for tarballs or zip
 # Don't test tarballs or zip in native test
-native_archive_list = [key for key in ext_archive.keys() if "tar" not in key and "zip" not in key]
+native_archive_list = [
+    key for key in ext_archive.keys() if "tar" not in key and "zip" not in key and "whl" not in key
+]
 
 
 @pytest.fixture
@@ -71,7 +73,9 @@ def test_native_unpacking(tmpdir_factory, archive_file_and_extension):
 
 @pytest.mark.not_on_windows("Only Python unpacking available on Windows")
 @pytest.mark.parametrize(
-    "archive_file_and_extension", [(ext, True) for ext in ext_archive.keys()], indirect=True
+    "archive_file_and_extension",
+    [(ext, True) for ext in ext_archive.keys() if "whl" not in ext],
+    indirect=True,
 )
 def test_system_unpacking(tmpdir_factory, archive_file_and_extension, compr_support_check):
     # actually run test

lib/spack/spack/util/compression.py

@@ -9,7 +9,7 @@
 import os
 import shutil
 import sys
-from typing import BinaryIO, Callable, Dict, List, Optional
+from typing import Any, BinaryIO, Callable, Dict, List, Optional
 
 import llnl.url
 from llnl.util import tty
@@ -157,6 +157,10 @@ def _system_gunzip(archive_file: str) -> str:
     return destination_abspath
 
 
+def _do_nothing(archive_file: str) -> None:
+    return None
+
+
 def _unzip(archive_file: str) -> str:
     """Returns path to extracted zip archive. Extract Zipfile, searching for unzip system
     executable. If unavailable, search for 'tar' executable on system and use instead.
@@ -283,26 +287,27 @@ def decompressor_for(path: str, extension: Optional[str] = None):
         return decompressor_for_nix(extension)
 
 
-def decompressor_for_nix(extension: str) -> Callable[[str], str]:
+def decompressor_for_nix(extension: str) -> Callable[[str], Any]:
     """Returns a function pointer to appropriate decompression algorithm based on extension type
     and unix specific considerations i.e. a reasonable expectation system utils like gzip, bzip2,
     and xz are available
 
     Args:
         extension: path of the archive file requiring decompression
     """
-    extension_to_decompressor: Dict[str, Callable[[str], str]] = {
+    extension_to_decompressor: Dict[str, Callable[[str], Any]] = {
         "zip": _unzip,
         "gz": _gunzip,
         "bz2": _bunzip2,
         "Z": _system_unZ,  # no builtin support for .Z files
         "xz": _lzma_decomp,
+        "whl": _do_nothing,
     }
 
     return extension_to_decompressor.get(extension, _system_untar)
 
 
-def _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[str], str]]:
+def _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[str], Any]]:
     """Returns appropriate python based decompression strategy
     based on extension type"""
     extension_to_decompressor: Dict[str, Callable[[str], str]] = {
@@ -313,7 +318,7 @@ def _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[
     return extension_to_decompressor.get(extension, None)
 
 
-def decompressor_for_win(extension: str) -> Callable[[str], str]:
+def decompressor_for_win(extension: str) -> Callable[[str], Any]:
     """Returns a function pointer to appropriate decompression
     algorithm based on extension type and Windows specific considerations
 
@@ -323,7 +328,7 @@ def decompressor_for_win(extension: str) -> Callable[[str], str]:
     and files as Python does not provide support for the UNIX compress algorithm
     """
     extension = llnl.url.expand_contracted_extension(extension)
-    extension_to_decompressor: Dict[str, Callable[[str], str]] = {
+    extension_to_decompressor: Dict[str, Callable[[str], Any]] = {
         # Windows native tar can handle .zip extensions, use standard unzip method
         "zip": _unzip,
         # if extension is standard tarball, invoke Windows native tar
@@ -333,6 +338,7 @@ def decompressor_for_win(extension: str) -> Callable[[str], str]:
         # detected
         "Z": _system_unZ,
         "xz": _lzma_decomp,
+        "whl": _do_nothing,
     }
 
     decompressor = extension_to_decompressor.get(extension)

var/spack/repos/builtin/packages/py-azureml-automl-core/package.py

@@ -14,11 +14,7 @@ class PyAzuremlAutomlCore(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_automl_core/azureml_automl_core-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="1fa4a900856b15e1ec9a6bb949946ed0c873a5a54da3db592f03dbb46a117ceb",
-        expand=False,
-    )
+    version("1.23.0", sha256="1fa4a900856b15e1ec9a6bb949946ed0c873a5a54da3db592f03dbb46a117ceb")
 
     depends_on("python@3.5:3", type=("build", "run"))
 

var/spack/repos/builtin/packages/py-azureml-core/package.py

@@ -17,21 +17,9 @@ class PyAzuremlCore(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="0965d0741e39cdb95cff5880dbf1a55fdd87cd9fc316884f965668e6cc36e628",
-        expand=False,
-    )
-    version(
-        "1.11.0",
-        sha256="df8a01b04bb156852480de0bdd78434ed84f386e1891752bdf887faeaa2ca417",
-        expand=False,
-    )
-    version(
-        "1.8.0",
-        sha256="a0f2b0977f18fb7dcb88c314594a4a85c636a36be3d582be1cae25655fea6105",
-        expand=False,
-    )
+    version("1.23.0", sha256="0965d0741e39cdb95cff5880dbf1a55fdd87cd9fc316884f965668e6cc36e628")
+    version("1.11.0", sha256="df8a01b04bb156852480de0bdd78434ed84f386e1891752bdf887faeaa2ca417")
+    version("1.8.0", sha256="a0f2b0977f18fb7dcb88c314594a4a85c636a36be3d582be1cae25655fea6105")
 
     depends_on("python@3.5:3.8", type=("build", "run"))
     depends_on("py-pytz", type=("build", "run"))

var/spack/repos/builtin/packages/py-azureml-dataprep-native/package.py

@@ -17,27 +17,23 @@ class PyAzuremlDataprepNative(PythonPackage):
         version(
             "30.0.0-py3.9",
             sha256="eaf3fcd9f965e87b03fe89d7c6fe6abce53483a79afc963e4981061f4c250e85",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_native/azureml_dataprep_native-30.0.0-cp39-cp39-macosx_10_9_x86_64.whl",
         )
         version(
             "30.0.0-py3.8",
             sha256="6772b638f9d03a041b17ce4343061f5d543019200904b9d361b2b2629c3595a7",
-            expand=False,
             preferred=True,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_native/azureml_dataprep_native-30.0.0-cp38-cp38-macosx_10_9_x86_64.whl",
         )
     elif sys.platform.startswith("linux"):
         version(
             "30.0.0-py3.9",
             sha256="b8673136948f682c84d047feacbfee436df053cba4f386f31c4c3a245a4e3646",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_native/azureml_dataprep_native-30.0.0-cp39-cp39-manylinux1_x86_64.whl",
         )
         version(
             "30.0.0-py3.8",
             sha256="d07cf20f22b14c98576e135bbad9bb8aaa3108941d2beaadf050b4238bc93a18",
-            expand=False,
             preferred=True,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_native/azureml_dataprep_native-30.0.0-cp38-cp38-manylinux1_x86_64.whl",
         )

var/spack/repos/builtin/packages/py-azureml-dataprep-rslex/package.py

@@ -20,54 +20,46 @@ class PyAzuremlDataprepRslex(PythonPackage):
         version(
             "1.9.0-py3.9",
             sha256="9bdaa31d129dac19ee20d5a3aad1726397e90d8d741b4f6de4554040800fefe8",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl",
         )
         version(
             "1.9.0-py3.8",
             sha256="9b2e741ac1c53d3f7e6061d264feccf157d97e404c772933a176e6021014484e",
-            expand=False,
             preferred=True,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl",
         )
 
         version(
             "1.8.0-py3.9",
             sha256="677c25a7e23ec7f91d25aa596f382f7f3b6d60fbc3258bead2b2a6aa42f3a16d",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl",
         )
         version(
             "1.8.0-py3.8",
             sha256="d7f2dec06296544b1707f5b01c6a4eaad744b4abfe9e8e89830b561c84d95a7a",
-            expand=False,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl",
         )
     elif sys.platform.startswith("linux"):
         version(
             "1.9.0-py3.9",
             sha256="79d52bb427e3ca781a645c4f11f7a8e5e2c8f61e61bfc162b4062d8e47bcf3d6",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.9.0-cp39-cp39-manylinux1_x86_64.whl",
         )
         version(
             "1.9.0-py3.8",
             sha256="a52461103b45867dd919bab593bb6f2426c9b5f5a435081e82a3c57c54c3add6",
-            expand=False,
             preferred=True,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.9.0-cp38-cp38-manylinux1_x86_64.whl",
         )
 
         version(
             "1.8.0-py3.9",
             sha256="e251a077669703ca117b157b225fbc20832169f913476cf79c01a5c6f8ff7a50",
-            expand=False,
             url="https://pypi.io/packages/cp39/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.8.0-cp39-cp39-manylinux1_x86_64.whl",
         )
         version(
             "1.8.0-py3.8",
             sha256="2ebfa164f0933a5cec383cd27ba10d33861a73237ef481ada5a9a822bb55514a",
-            expand=False,
             url="https://pypi.io/packages/cp38/a/azureml_dataprep_rslex/azureml_dataprep_rslex-1.8.0-cp38-cp38-manylinux1_x86_64.whl",
         )
 

var/spack/repos/builtin/packages/py-azureml-dataprep/package.py

@@ -13,16 +13,8 @@ class PyAzuremlDataprep(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/python/api/overview/azure/ml/?view=azure-ml-py"
     url = "https://pypi.io/packages/py3/a/azureml_dataprep/azureml_dataprep-2.0.2-py3-none-any.whl"
 
-    version(
-        "2.11.0",
-        sha256="755c0d7cfe228705aee7adc97813fb6d7d6ecb048b66f47c1fd5897f2709c3a2",
-        expand=False,
-    )
-    version(
-        "2.10.1",
-        sha256="a36f807112ff1e64d21265b8e7f40154c93e3bead539e2a74c9d74200fd77c86",
-        expand=False,
-    )
+    version("2.11.0", sha256="755c0d7cfe228705aee7adc97813fb6d7d6ecb048b66f47c1fd5897f2709c3a2")
+    version("2.10.1", sha256="a36f807112ff1e64d21265b8e7f40154c93e3bead539e2a74c9d74200fd77c86")
 
     variant("fuse", default=False, description="Build with FUSE support")
 

var/spack/repos/builtin/packages/py-azureml-dataset-runtime/package.py

@@ -14,11 +14,7 @@ class PyAzuremlDatasetRuntime(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml-dataset-runtime/azureml_dataset_runtime-1.11.0.post1-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="96ca73d03ffedc0dd336d9383d2e17cf74548a89fc7ca4c201c599817c97bbc6",
-        expand=False,
-    )
+    version("1.23.0", sha256="96ca73d03ffedc0dd336d9383d2e17cf74548a89fc7ca4c201c599817c97bbc6")
 
     variant("fuse", default=False, description="Build with FUSE support")
 

var/spack/repos/builtin/packages/py-azureml-pipeline-core/package.py

@@ -13,21 +13,9 @@ class PyAzuremlPipelineCore(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_pipeline_core/azureml_pipeline_core-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="347e3e41559879611d53eeff5c05dd133db6fa537edcf2b9f70d91aad461df02",
-        expand=False,
-    )
-    version(
-        "1.11.0",
-        sha256="98012195e3bba12bf42ac69179549038b3563b39e3dadab4f1d06407a00ad8b3",
-        expand=False,
-    )
-    version(
-        "1.8.0",
-        sha256="24e1c57a57e75f9d74ea6f45fa4e93c1ee3114c8ed9029d538f9cc8e4f8945b2",
-        expand=False,
-    )
+    version("1.23.0", sha256="347e3e41559879611d53eeff5c05dd133db6fa537edcf2b9f70d91aad461df02")
+    version("1.11.0", sha256="98012195e3bba12bf42ac69179549038b3563b39e3dadab4f1d06407a00ad8b3")
+    version("1.8.0", sha256="24e1c57a57e75f9d74ea6f45fa4e93c1ee3114c8ed9029d538f9cc8e4f8945b2")
 
     depends_on("python@3.5:3", type=("build", "run"))
     depends_on("py-azureml-core@1.23.0:1.23", when="@1.23.0", type=("build", "run"))

var/spack/repos/builtin/packages/py-azureml-pipeline-steps/package.py

@@ -13,11 +13,7 @@ class PyAzuremlPipelineSteps(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_pipeline_steps/azureml_pipeline_steps-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="72154c2f75624a1e7500b8e2239ae1354eeedf66d2cabb11e213b7eb80aedddb",
-        expand=False,
-    )
+    version("1.23.0", sha256="72154c2f75624a1e7500b8e2239ae1354eeedf66d2cabb11e213b7eb80aedddb")
 
     depends_on("python@3:", type=("build", "run"))
 

var/spack/repos/builtin/packages/py-azureml-pipeline/package.py

@@ -16,11 +16,7 @@ class PyAzuremlPipeline(PythonPackage):
         "https://pypi.io/packages/py3/a/azureml_pipeline/azureml_pipeline-1.11.0-py3-none-any.whl"
     )
 
-    version(
-        "1.23.0",
-        sha256="ed0fae96771840d3ffd63d63df1b1eed2f50c3b8dbe7b672a4f1ba6e66d0a392",
-        expand=False,
-    )
+    version("1.23.0", sha256="ed0fae96771840d3ffd63d63df1b1eed2f50c3b8dbe7b672a4f1ba6e66d0a392")
 
     depends_on("python@3:", type=("build", "run"))
 

var/spack/repos/builtin/packages/py-azureml-sdk/package.py

@@ -15,11 +15,7 @@ class PyAzuremlSdk(PythonPackage):
 
     maintainers("adamjstewart")
 
-    version(
-        "1.23.0",
-        sha256="b9520f426831acb99fafa1ecd154b6bfd4f73fbf71e918d819f9db4a75438ab9",
-        expand=False,
-    )
+    version("1.23.0", sha256="b9520f426831acb99fafa1ecd154b6bfd4f73fbf71e918d819f9db4a75438ab9")
 
     # https://github.com/Azure/MachineLearningNotebooks/issues/1285
     depends_on("python@3.5:3.8", type=("build", "run"))

var/spack/repos/builtin/packages/py-azureml-telemetry/package.py

@@ -14,21 +14,9 @@ class PyAzuremlTelemetry(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_telemetry/azureml_telemetry-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="68f9aac77e468db80e60f75d0843536082e2884ab251b6d3054dd623bd9c9e0d",
-        expand=False,
-    )
-    version(
-        "1.11.0",
-        sha256="0d46c4a7bb8c0b188f1503504a6029384bc2237d82a131e7d1e9e89c3491b1fc",
-        expand=False,
-    )
-    version(
-        "1.8.0",
-        sha256="de657efe9773bea0de76c432cbab34501ac28606fe1b380d6883562ebda3d804",
-        expand=False,
-    )
+    version("1.23.0", sha256="68f9aac77e468db80e60f75d0843536082e2884ab251b6d3054dd623bd9c9e0d")
+    version("1.11.0", sha256="0d46c4a7bb8c0b188f1503504a6029384bc2237d82a131e7d1e9e89c3491b1fc")
+    version("1.8.0", sha256="de657efe9773bea0de76c432cbab34501ac28606fe1b380d6883562ebda3d804")
 
     depends_on("python@3.5:3", type=("build", "run"))
     depends_on("py-applicationinsights", type=("build", "run"))

var/spack/repos/builtin/packages/py-azureml-train-automl-client/package.py

@@ -15,11 +15,7 @@ class PyAzuremlTrainAutomlClient(PythonPackage):
     homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/"
     url = "https://pypi.io/packages/py3/a/azureml_train_automl_client/azureml_train_automl_client-1.11.0-py3-none-any.whl"
 
-    version(
-        "1.23.0",
-        sha256="ac5f1ce9b04b4e61e2e28e0fa8d2d8e47937a546f624d1cd3aa6bc4f9110ecbe",
-        expand=False,
-    )
+    version("1.23.0", sha256="ac5f1ce9b04b4e61e2e28e0fa8d2d8e47937a546f624d1cd3aa6bc4f9110ecbe")
 
     depends_on("python@3.5:3", type=("build", "run"))