From ff634c41671f57e6d77413de2c5474d58ec9e576 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 03:52:54 -0700 Subject: [PATCH 1/8] fix: support legacy arm platforms --- .../workflows/python_pip/packager.py | 3 ++ .../workflows/python_pip/test_python_pip.py | 35 +++++++++++++++++++ .../testdata/requirements-legacy.txt | 1 + 3 files changed, 39 insertions(+) create mode 100644 tests/integration/workflows/python_pip/testdata/requirements-legacy.txt diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index c92463dc6..c32033042 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -192,6 +192,9 @@ class DependencyBuilder(object): "manylinux1_x86_64": "manylinux_2_5_x86_64", "manylinux2010_x86_64": "manylinux_2_12_x86_64", "manylinux2014_x86_64": "manylinux_2_17_x86_64", + "manylinux1_aarch64": "manylinux_2_5_aarch64", + "manylinux2010_aarch64": "manylinux_2_12_aarch64", + "manylinux2014_aarch64": "manylinux_2_17_aarch64", } _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index a5b19f620..88e0b90a9 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -37,6 +37,7 @@ def setUp(self): self.dependencies_dir = tempfile.mkdtemp() self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt") + self.manifest_path_legacy = os.path.join(self.TEST_DATA_FOLDER, "requirements-legacy.txt") self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt") self.manifest_path_sdist = os.path.join(self.TEST_DATA_FOLDER, "requirements-wrapt.txt") @@ -44,6 +45,7 @@ def setUp(self): "__init__.py", "main.py", "requirements-invalid.txt", + "requirements-legacy.txt", "requirements-numpy.txt", "requirements-wrapt.txt", "local-dependencies", @@ -103,6 +105,20 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_must_build_python_project_for_legacy_platforms(self): + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_legacy, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + ) + + expected_files = self.test_data_files.union({"inflate64", "inflate64.libs", "inflate64-0.1.4.dist-info"}) + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + def test_must_build_python_project_python3_binary(self): python_paths = which("python") with tempfile.TemporaryDirectory() as executable_dir_str: @@ -175,6 +191,25 @@ def test_must_build_python_project_with_arm_architecture(self): else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) + def test_must_build_python_project_with_arm_architecture_for_legacy_platforms(self): + if self.runtime not in ARM_RUNTIMES: + self.skipTest("{} is not supported on ARM architecture".format(self.runtime)) + ### Check the wheels + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_legacy, + runtime=self.runtime, + architecture="arm64", + experimental_flags=self.experimental_flags, + ) + expected_files = self.test_data_files.union({"inflate64", "inflate64-0.1.4.dist-info"}) + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + + self.check_architecture_in("inflate64-0.1.4.dist-info", ["manylinux2014_aarch64"]) + def test_mismatch_runtime_python_project(self): # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python3.7 # is still accessible within a python 3.8 virtualenv. diff --git a/tests/integration/workflows/python_pip/testdata/requirements-legacy.txt b/tests/integration/workflows/python_pip/testdata/requirements-legacy.txt new file mode 100644 index 000000000..08f90227a --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/requirements-legacy.txt @@ -0,0 +1 @@ +inflate64==0.1.4 \ No newline at end of file From 466afc4f9703b774f7e086bd683c827fc9ee05c2 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 04:12:18 -0700 Subject: [PATCH 2/8] fix testing --- .../workflows/python_pip/test_python_pip.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 88e0b90a9..a54792e66 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -105,6 +105,7 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_build_python_project_for_legacy_platforms(self): self.builder.build( self.source_dir, @@ -115,7 +116,18 @@ def test_must_build_python_project_for_legacy_platforms(self): experimental_flags=self.experimental_flags, ) - expected_files = self.test_data_files.union({"inflate64", "inflate64.libs", "inflate64-0.1.4.dist-info"}) + expected_files = self.test_data_files.union( + { + "inflate64", + "inflate64-0.1.4.dist-info", + "typing_extensions.py", + "importlib_metadata-6.1.0.dist-info", + "typing_extensions-4.5.0.dist-info", + "zipp-3.15.0.dist-info", + "importlib_metadata", + "zipp", + } + ) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -191,6 +203,7 @@ def test_must_build_python_project_with_arm_architecture(self): else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_build_python_project_with_arm_architecture_for_legacy_platforms(self): if self.runtime not in ARM_RUNTIMES: self.skipTest("{} is not supported on ARM architecture".format(self.runtime)) @@ -204,7 +217,18 @@ def test_must_build_python_project_with_arm_architecture_for_legacy_platforms(se architecture="arm64", experimental_flags=self.experimental_flags, ) - expected_files = self.test_data_files.union({"inflate64", "inflate64-0.1.4.dist-info"}) + expected_files = self.test_data_files.union( + { + "inflate64", + "inflate64-0.1.4.dist-info", + "typing_extensions.py", + "importlib_metadata-6.1.0.dist-info", + "typing_extensions-4.5.0.dist-info", + "zipp-3.15.0.dist-info", + "importlib_metadata", + "zipp", + } + ) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) From 12b9dc3f3e9ace41d2c8b054e4b1d481b7646dbb Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 04:31:16 -0700 Subject: [PATCH 3/8] fix testing --- .../workflows/python_pip/test_python_pip.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index a54792e66..2bd7511f4 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -116,18 +116,28 @@ def test_must_build_python_project_for_legacy_platforms(self): experimental_flags=self.experimental_flags, ) - expected_files = self.test_data_files.union( - { - "inflate64", - "inflate64-0.1.4.dist-info", - "typing_extensions.py", - "importlib_metadata-6.1.0.dist-info", - "typing_extensions-4.5.0.dist-info", - "zipp-3.15.0.dist-info", - "importlib_metadata", - "zipp", - } - ) + if self.runtime == "python3.7": + expected_files = self.test_data_files.union( + { + "inflate64", + "inflate64.libs", + "inflate64-0.1.4.dist-info", + "typing_extensions.py", + "importlib_metadata-6.1.0.dist-info", + "typing_extensions-4.5.0.dist-info", + "zipp-3.15.0.dist-info", + "importlib_metadata", + "zipp", + } + ) + else: + expected_files = self.test_data_files.union( + { + "inflate64", + "inflate64.libs", + "inflate64-0.1.4.dist-info", + } + ) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) From a16cb5c70ad53760f135bdb3420208d186d3d0ff Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 04:39:39 -0700 Subject: [PATCH 4/8] testing --- aws_lambda_builders/workflows/python_pip/packager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index c32033042..09dd08700 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -432,6 +432,7 @@ def _is_compatible_platform_tag(self, expected_abi, platform): # Verify the tag pattern # Try to get the matching value for legacy values or keep the current perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) + print(f">>>>>> {platform}, {perennial_tag}") match = re.match("manylinux_([0-9]+)_([0-9]+)_" + arch, perennial_tag) if match is None: From 165d142d852836c2ff6d4836adf26e6dcba86d64 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 04:48:18 -0700 Subject: [PATCH 5/8] testing --- aws_lambda_builders/workflows/python_pip/packager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 09dd08700..767420598 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -424,6 +424,7 @@ def _is_compatible_platform_tag(self, expected_abi, platform): In addition to checking the tag pattern, we also need to verify the glibc version """ + print(f">>>>>> expected_abi {expected_abi}, {platform}") if platform in self._COMPATIBLE_PLATFORMS[self.architecture]: return True From 2abcb753098e27c4bdc2cf6398bc2dcddbfbc748 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 04:54:25 -0700 Subject: [PATCH 6/8] testing --- aws_lambda_builders/workflows/python_pip/packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 767420598..cc7554104 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -298,6 +298,7 @@ def _download_dependencies(self, directory, requirements_filename): incompatible_wheels = set() sdists = set() for package in deps: + print(f">>>>> package.filename {package.filename}") if package.dist_type == "sdist": sdists.add(package) elif self._is_compatible_wheel_filename(package.filename): @@ -424,7 +425,6 @@ def _is_compatible_platform_tag(self, expected_abi, platform): In addition to checking the tag pattern, we also need to verify the glibc version """ - print(f">>>>>> expected_abi {expected_abi}, {platform}") if platform in self._COMPATIBLE_PLATFORMS[self.architecture]: return True From 4125e8ec41d2bd9f5f479d0d80bd63b41e8cbb21 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 05:35:34 -0700 Subject: [PATCH 7/8] testing --- tests/integration/workflows/python_pip/test_python_pip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 2bd7511f4..13dbdd211 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -105,7 +105,7 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) - @skipIf(IS_WINDOWS, "Skip in windows tests") + @skipIf(IS_WINDOWS or NOT_ARM, "Skip in windows tests") def test_must_build_python_project_for_legacy_platforms(self): self.builder.build( self.source_dir, @@ -213,7 +213,7 @@ def test_must_build_python_project_with_arm_architecture(self): else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) - @skipIf(IS_WINDOWS, "Skip in windows tests") + @skipIf(IS_WINDOWS or NOT_ARM, "Skip in windows tests") def test_must_build_python_project_with_arm_architecture_for_legacy_platforms(self): if self.runtime not in ARM_RUNTIMES: self.skipTest("{} is not supported on ARM architecture".format(self.runtime)) From 1bc0215aaf9da932615aef3dc47daff7684a72b6 Mon Sep 17 00:00:00 2001 From: Mohamed ElAsmar Date: Wed, 29 Mar 2023 05:36:20 -0700 Subject: [PATCH 8/8] testing --- tests/integration/workflows/python_pip/test_python_pip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 13dbdd211..b1681dd4c 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -105,7 +105,7 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) - @skipIf(IS_WINDOWS or NOT_ARM, "Skip in windows tests") + @skipIf(NOT_ARM, "Skip in windows tests") def test_must_build_python_project_for_legacy_platforms(self): self.builder.build( self.source_dir, @@ -213,7 +213,7 @@ def test_must_build_python_project_with_arm_architecture(self): else: self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"]) - @skipIf(IS_WINDOWS or NOT_ARM, "Skip in windows tests") + @skipIf(NOT_ARM, "Skip in windows tests") def test_must_build_python_project_with_arm_architecture_for_legacy_platforms(self): if self.runtime not in ARM_RUNTIMES: self.skipTest("{} is not supported on ARM architecture".format(self.runtime))