From 52bd0eefa7288ffacf7c91ae995e39bd3453dfda Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 07:42:04 +0200 Subject: [PATCH 01/13] Claim all existing packages on pypi --- .github/workflows/claim-pypi-name.yaml | 39 ++++++++++++++++++++++++++ .github/workflows/scripts/build.sh | 29 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .github/workflows/claim-pypi-name.yaml create mode 100755 .github/workflows/scripts/build.sh diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml new file mode 100644 index 0000000000000..1088c6e3c3d76 --- /dev/null +++ b/.github/workflows/claim-pypi-name.yaml @@ -0,0 +1,39 @@ +name: build ddev + +on: + pull_request: + branches: + - master + +defaults: + run: + shell: bash + + +jobs: + python-artifacts: + name: Build wheel and source distribution + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install Build + run: pip install -U build[virtualenv] + + - name: build packages + run: | + bash .github/workflows/scripts/build.sh + + # - name: Push Python artifacts to PyPI + # uses: pypa/gh-action-pypi-publish@v1.8.14 + # with: + # skip-existing: true + # user: __token__ + # password: ${{ secrets.INTEGRATIONS_PYPI_NAME_CLAIM }} diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build.sh new file mode 100755 index 0000000000000..14767527a2905 --- /dev/null +++ b/.github/workflows/scripts/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Requires python 3.11+ (for tomllib) and the `build` package. +# Run this script from the root of the repo. +# The TL;DR of this script is: for every subdirectory with a pyproject.toml file we build an empty Python wheel. + +mkdir -p dist +mkdir -p pkg_placeholder +for file_or_subdir in *; do + if [[ -d "${file_or_subdir}" ]]; then + pyproject="${file_or_subdir}/pyproject.toml" + if [[ -f "${pyproject}" ]]; then + pypi_pkg_name=$(python -c "import tomllib, pathlib; contents = pathlib.Path('${pyproject}').read_text(); data = tomllib.loads(contents); print(data['project']['name'])") + # multiline strings are sensitive to indentation, so we must unindent the following command +cat < pkg_placeholder/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +[project] +name = "${pypi_pkg_name}" +version = "0.0.1" +[tool.hatch.build.targets.wheel] +bypass-selection = true +EOF + python -m build --wheel pkg_placeholder + mv pkg_placeholder/dist/* dist/ + fi + fi +done From 815311db48667d190f6ec7945936f048a013f702 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 08:01:45 +0200 Subject: [PATCH 02/13] better script and job name --- .github/workflows/claim-pypi-name.yaml | 6 +++--- .../workflows/scripts/{build.sh => build_placeholders.sh} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/scripts/{build.sh => build_placeholders.sh} (100%) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 1088c6e3c3d76..6ea985f6c204f 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -1,4 +1,4 @@ -name: build ddev +name: Build Placeholder PyPI Packages on: pull_request: @@ -27,9 +27,9 @@ jobs: - name: Install Build run: pip install -U build[virtualenv] - - name: build packages + - name: Build Packages run: | - bash .github/workflows/scripts/build.sh + bash .github/workflows/scripts/build_placeholders.sh # - name: Push Python artifacts to PyPI # uses: pypa/gh-action-pypi-publish@v1.8.14 diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build_placeholders.sh similarity index 100% rename from .github/workflows/scripts/build.sh rename to .github/workflows/scripts/build_placeholders.sh From ffb0b222e044921611c64681847e4d13f7514499 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 22:39:32 +0200 Subject: [PATCH 03/13] push wheels to pypi --- .github/workflows/claim-pypi-name.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 6ea985f6c204f..f4badf668067c 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -31,9 +31,9 @@ jobs: run: | bash .github/workflows/scripts/build_placeholders.sh - # - name: Push Python artifacts to PyPI - # uses: pypa/gh-action-pypi-publish@v1.8.14 - # with: - # skip-existing: true - # user: __token__ - # password: ${{ secrets.INTEGRATIONS_PYPI_NAME_CLAIM }} + - name: Push Python artifacts to PyPI + uses: pypa/gh-action-pypi-publish@v1.8.14 + with: + skip-existing: true + user: __token__ + password: ${{ secrets.INTEGRATIONS_PYPI_NAME_CLAIM }} From 6660e078f2848bc5c52acece1c54fabd8beda4cd Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 22:59:07 +0200 Subject: [PATCH 04/13] Don't validate metadata and increase verbosity for better feedback --- .github/workflows/claim-pypi-name.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index f4badf668067c..96795cdcf9124 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -34,6 +34,9 @@ jobs: - name: Push Python artifacts to PyPI uses: pypa/gh-action-pypi-publish@v1.8.14 with: + # We don't mind invalid metadata, we only want to claim the package name. + verify-metadata: false + verbose: true skip-existing: true user: __token__ password: ${{ secrets.INTEGRATIONS_PYPI_NAME_CLAIM }} From ab9410951e4fc1ddf7391455c5818216e71b4bc3 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 23:36:35 +0200 Subject: [PATCH 05/13] Drop build isolation to speed up job --- .github/workflows/scripts/build_placeholders.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/build_placeholders.sh b/.github/workflows/scripts/build_placeholders.sh index 14767527a2905..0ca5c907bb3c5 100755 --- a/.github/workflows/scripts/build_placeholders.sh +++ b/.github/workflows/scripts/build_placeholders.sh @@ -22,7 +22,10 @@ version = "0.0.1" [tool.hatch.build.targets.wheel] bypass-selection = true EOF - python -m build --wheel pkg_placeholder + # We only want wheels. + # We don't need build isolation because we'll trash the env anyway in CI. + # Skipping isolation speeds up the job. + python -m build --no-isolation --wheel pkg_placeholder mv pkg_placeholder/dist/* dist/ fi fi From be9a15ecd8912f51245157ffa2296b9573aa322d Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 16 May 2024 23:44:18 +0200 Subject: [PATCH 06/13] install hatchling separately --- .github/workflows/claim-pypi-name.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 96795cdcf9124..7390132280048 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -24,8 +24,8 @@ jobs: with: python-version: 3.11 - - name: Install Build - run: pip install -U build[virtualenv] + - name: Install Build Deps + run: pip install -U build[virtualenv] hatchling - name: Build Packages run: | From 9cb9ae4bf3b42426c5e777284de5cc52e716c75a Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Fri, 17 May 2024 08:19:12 +0200 Subject: [PATCH 07/13] Skip package that's blocked on pypi --- .github/workflows/scripts/build_placeholders.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/scripts/build_placeholders.sh b/.github/workflows/scripts/build_placeholders.sh index 0ca5c907bb3c5..b7b9a7f7a85b9 100755 --- a/.github/workflows/scripts/build_placeholders.sh +++ b/.github/workflows/scripts/build_placeholders.sh @@ -11,6 +11,11 @@ for file_or_subdir in *; do pyproject="${file_or_subdir}/pyproject.toml" if [[ -f "${pyproject}" ]]; then pypi_pkg_name=$(python -c "import tomllib, pathlib; contents = pathlib.Path('${pyproject}').read_text(); data = tomllib.loads(contents); print(data['project']['name'])") + # Remove these clauses when they become unblocked. + if [[ "${pypi_pkg_name}" = "datadog-cert-manager"]]; then + echo "Skipping ${pypi_pkg_name} until it gets unblocked on PyPI" + continue + fi # multiline strings are sensitive to indentation, so we must unindent the following command cat < pkg_placeholder/pyproject.toml [build-system] From ace4b04f6e6fa5beb81bc185d84c43151a7b1338 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Fri, 17 May 2024 08:26:11 +0200 Subject: [PATCH 08/13] fix bash syntax error --- .github/workflows/scripts/build_placeholders.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/build_placeholders.sh b/.github/workflows/scripts/build_placeholders.sh index b7b9a7f7a85b9..de7c998d45a94 100755 --- a/.github/workflows/scripts/build_placeholders.sh +++ b/.github/workflows/scripts/build_placeholders.sh @@ -12,7 +12,7 @@ for file_or_subdir in *; do if [[ -f "${pyproject}" ]]; then pypi_pkg_name=$(python -c "import tomllib, pathlib; contents = pathlib.Path('${pyproject}').read_text(); data = tomllib.loads(contents); print(data['project']['name'])") # Remove these clauses when they become unblocked. - if [[ "${pypi_pkg_name}" = "datadog-cert-manager"]]; then + if [[ "${pypi_pkg_name}" = "datadog-cert-manager" ]]; then echo "Skipping ${pypi_pkg_name} until it gets unblocked on PyPI" continue fi From 0d52e98a3191b9a9ec714bf690598c7d03203ab8 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Fri, 17 May 2024 16:27:18 +0200 Subject: [PATCH 09/13] upload cert manager, it's been unblocked --- .github/workflows/scripts/build_placeholders.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/scripts/build_placeholders.sh b/.github/workflows/scripts/build_placeholders.sh index de7c998d45a94..0ca5c907bb3c5 100755 --- a/.github/workflows/scripts/build_placeholders.sh +++ b/.github/workflows/scripts/build_placeholders.sh @@ -11,11 +11,6 @@ for file_or_subdir in *; do pyproject="${file_or_subdir}/pyproject.toml" if [[ -f "${pyproject}" ]]; then pypi_pkg_name=$(python -c "import tomllib, pathlib; contents = pathlib.Path('${pyproject}').read_text(); data = tomllib.loads(contents); print(data['project']['name'])") - # Remove these clauses when they become unblocked. - if [[ "${pypi_pkg_name}" = "datadog-cert-manager" ]]; then - echo "Skipping ${pypi_pkg_name} until it gets unblocked on PyPI" - continue - fi # multiline strings are sensitive to indentation, so we must unindent the following command cat < pkg_placeholder/pyproject.toml [build-system] From d8445d77514ed7f32e26226f16ed8ffc3573527b Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 30 May 2024 17:42:34 +0200 Subject: [PATCH 10/13] Job runs on schedule --- .github/workflows/claim-pypi-name.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 7390132280048..7a12a9aa7898c 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -1,9 +1,10 @@ name: Build Placeholder PyPI Packages on: - pull_request: - branches: - - master + schedule: + # At 3AM UTC + # Running this every night strikes a good balance between claiming names fast without spamming PyPI with requests. + - cron: "0 3 * * *" defaults: run: @@ -37,6 +38,7 @@ jobs: # We don't mind invalid metadata, we only want to claim the package name. verify-metadata: false verbose: true + # Only uploading the missing wheels makes this job idempotent and reduces its complexity. skip-existing: true user: __token__ password: ${{ secrets.INTEGRATIONS_PYPI_NAME_CLAIM }} From 81cb5426ea48df95a6581364d284d6aeb487cdbe Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Tue, 4 Jun 2024 10:03:02 +0200 Subject: [PATCH 11/13] add comment with motivation; allow workflow to be triggered manually --- .github/workflows/claim-pypi-name.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 7a12a9aa7898c..45fe29fd44b30 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -1,10 +1,15 @@ +# This workflow claims package names on PyPI for our integrations by publishing empty packages. +# The working packages can be found here: +# https://dd-integrations-core-wheels-build-stable.datadoghq.com/targets/simple/index.html +# This is a work-around until PyPI adds support for namespaces and we claim an entire namespace for Datadog. name: Build Placeholder PyPI Packages on: + workflow_dispatch: schedule: - # At 3AM UTC - # Running this every night strikes a good balance between claiming names fast without spamming PyPI with requests. - - cron: "0 3 * * *" + # At 3AM UTC + # Running this every night strikes a good balance between claiming names fast without spamming PyPI with requests. + - cron: "0 3 * * *" defaults: run: From f2c16561673e12fa9aecd6ebf07f0b03fb227ddd Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Tue, 4 Jun 2024 14:24:35 +0200 Subject: [PATCH 12/13] Update .github/workflows/claim-pypi-name.yaml Co-authored-by: Ofek Lev --- .github/workflows/claim-pypi-name.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index 45fe29fd44b30..a362958c85614 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -38,7 +38,7 @@ jobs: bash .github/workflows/scripts/build_placeholders.sh - name: Push Python artifacts to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.14 + uses: pypa/gh-action-pypi-publish@release/v1 with: # We don't mind invalid metadata, we only want to claim the package name. verify-metadata: false From 1c65ee383f6c7cb253668ec7c71233e8807ca315 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Tue, 4 Jun 2024 14:26:32 +0200 Subject: [PATCH 13/13] Update .github/workflows/claim-pypi-name.yaml Co-authored-by: Ofek Lev --- .github/workflows/claim-pypi-name.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/claim-pypi-name.yaml b/.github/workflows/claim-pypi-name.yaml index a362958c85614..052fb8a2d1137 100644 --- a/.github/workflows/claim-pypi-name.yaml +++ b/.github/workflows/claim-pypi-name.yaml @@ -18,7 +18,7 @@ defaults: jobs: python-artifacts: - name: Build wheel and source distribution + name: Build wheel runs-on: ubuntu-latest steps: