Skip to content

Commit

Permalink
Remove obsolete binary workflows, limit linux to 1 python version (#1447
Browse files Browse the repository at this point in the history
) (#1448)

* Remove obsolete binary workflows, limit linux to 1 python version

* fixes

* fix

* fix

Co-authored-by: Andrey Talman <atalman@fb.com>
  • Loading branch information
ramanishsingh and atalman authored Feb 20, 2025
1 parent c4177af commit 377e64c
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 217 deletions.
175 changes: 2 additions & 173 deletions .github/workflows/_build_test_upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,6 @@ on:
pytorch_version:
required: true
type: string
do-upload:
required: false
default: true
type: boolean
secrets:
PYPI_TOKEN:
required: false

permissions:
id-token: write
Expand Down Expand Up @@ -47,173 +40,9 @@ jobs:
echo "type=$RELEASE_TYPE" >> $GITHUB_OUTPUT
id: get_release_type

wheel_build_test:
needs: get_release_type
runs-on: ${{ matrix.os }}
container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }}
strategy:
fail-fast: false
matrix:
os:
- windows-latest
python-version:
- 3.9
- "3.10"
- "3.11"
- "3.12"
- "3.13"
- pure
exclude:
- os: windows-latest
python-version: pure
steps:
- name: Checkout Source Repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
submodules: recursive
- name: Setup Python ${{ matrix.python-version }} for Windows
if: ${{ startsWith( matrix.os, 'windows' ) }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Add temp runner environment variables
if: ${{ startsWith( matrix.os, 'macos' ) }}
shell: bash -l {0}
run: |
echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
- name: Setup msbuild on Windows
if: startsWith( matrix.os, 'windows' )
uses: microsoft/setup-msbuild@v1.1
- name: Set up Visual Studio shell
if: startsWith( matrix.os, 'windows' )
uses: egor-tensin/vs-shell@v2
with:
arch: x64
- name: Install Build Dependency
shell: bash -l {0}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
pip install cmake ninja setuptools
echo "/home/runner/.local/bin" >> $GITHUB_PATH
- name: Install PyTorch and Build TorchData Wheel
shell: bash -l {0}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
PYTORCH_VERSION: ${{ inputs.pytorch_version }}
BUILD_S3: 0
run: |
set -ex
packaging/build_wheel.sh
- name: Validate TorchData Wheel
shell: bash -l {0}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
pip3 install pkginfo
for pkg in dist/torchdata*.whl; do
echo "PkgInfo of $pkg:"
pkginfo $pkg
if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then
auditwheel show $pkg
auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse
fi
done
- name: Install TorchData Wheel
shell: bash -l {0}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
pip3 install dist/torchdata*.whl
- name: Run Smoke Tests
shell: bash -l {0}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
if ${{ matrix.python-version == 'pure' }}; then
python test/smoke_test/smoke_test.py --no-s3
else
python test/smoke_test/smoke_test.py
fi
- name: Upload Wheels to Github
if: always()
uses: actions/upload-artifact@v4
with:
name: torchdata-artifact-${{ matrix.os }}-${{ matrix.python-version }}
path: dist/torchdata*.whl

wheel_upload:
if: always() && inputs.branch != '' && inputs.do-upload == true
needs: [get_release_type, wheel_build_test]
runs-on: ubuntu-latest
environment: pytorchbot-env
outputs:
upload: ${{ steps.trigger_upload.outputs.value }}
steps:
- name: Configure aws credentials (pytorch account)
if: ${{ needs.get_release_type.outputs.type == 'nightly' }}
uses: aws-actions/configure-aws-credentials@v3
with:
role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels
aws-region: us-east-1

- name: Configure aws credentials (pytorch account)
if: ${{ needs.get_release_type.outputs.type == 'test' }}
uses: aws-actions/configure-aws-credentials@v3
with:
role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels
aws-region: us-east-1
- name: Download Artifacts from Github
continue-on-error: true
uses: actions/download-artifact@v4
with:
pattern: torchdata-artifact-*
merge-multiple: true
- name: Determine if Wheel Uploading is needed
run: |
upload=false
for txt in torchdata*.whl; do
upload=true
break
done
echo "value=$upload" >> $GITHUB_OUTPUT
id: trigger_upload
- name: Display All TorchData Wheels
if: steps.trigger_upload.outputs.value == 'true'
run: ls -lh torchdata*.whl
- name: Upload Wheels to S3 Storage
if: steps.trigger_upload.outputs.value == 'true'
run: |
if [[ ${{ inputs.branch }} == 'main' ]]; then
S3_PATH=s3://pytorch/whl/nightly/
else
S3_PATH=s3://pytorch/whl/test/
fi
pip3 install awscli
set -x
for pkg in torchdata*.whl; do
aws s3 cp "$pkg" "$S3_PATH" --acl public-read
done
- name: Upload Official Wheels to PYPI
if: |
steps.trigger_upload.outputs.value == 'true' &&
needs.get_release_type.outputs.type == 'official'
env:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
run: |
pip3 install twine
python -m twine upload \
--username __token__ \
--password "$PYPI_TOKEN" \
torchdata*.whl
build_docs:
if: |
always() && inputs.branch != '' &&
( needs.wheel_upload.outputs.upload == 'true')
needs: [get_release_type, wheel_upload]
if: always() && inputs.branch != ''
needs: get_release_type
runs-on: ubuntu-latest
steps:
- name: Setup Python 3.9
Expand Down
44 changes: 0 additions & 44 deletions .github/workflows/build-wheels-m1.yml

This file was deleted.

1 change: 1 addition & 0 deletions .github/workflows/build_wheels_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
test-infra-ref: main
with-cuda: disable
with-rocm: disable
python-versions: '["3.9"]'
build:
needs: generate-matrix
strategy:
Expand Down

0 comments on commit 377e64c

Please sign in to comment.