Skip to content

Commit ea23621

Browse files
BordaBorda
authored
Merge branch 'master' into fix-neptune-checkpoint-extension
2 parents 096db84 + 41315bc commit ea23621

File tree

86 files changed

+735
-732
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+735
-732
lines changed

‎.azure/gpu-benchmarks.yml

+1
Original file line numberDiff line numberDiff line change
@@ -108,5 +108,6 @@ jobs:
108108
condition: and(succeeded(), eq(variables['PACKAGE_NAME'], 'fabric'))
109109
env:
110110
PL_RUN_CUDA_TESTS: "1"
111+
PL_RUN_STANDALONE_TESTS: "1"
111112
displayName: "Testing: fabric standalone tasks"
112113
timeoutInMinutes: "10"

‎.azure/gpu-tests-fabric.yml

+13-7
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,14 @@ jobs:
5656
options: "--gpus=all --shm-size=2gb -v /var/tmp:/var/tmp"
5757
strategy:
5858
matrix:
59+
"Fabric | oldest":
60+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
61+
PACKAGE_NAME: "fabric"
5962
"Fabric | latest":
60-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.11-torch2.3-cuda12.1.0"
63+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.6-cuda12.4.1"
6164
PACKAGE_NAME: "fabric"
6265
"Lightning | latest":
63-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
66+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.6-cuda12.4.1"
6467
PACKAGE_NAME: "lightning"
6568
workspace:
6669
clean: all
@@ -77,9 +80,8 @@ jobs:
7780
displayName: "set env. vars"
7881
- bash: |
7982
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
80-
echo "##vso[task.setvariable variable=TORCHVISION_URL]https://download.pytorch.org/whl/test/cu124/torchvision-0.19.0%2Bcu124-cp${PYTHON_VERSION_MM}-cp${PYTHON_VERSION_MM}-linux_x86_64.whl"
8183
condition: endsWith(variables['Agent.JobName'], 'future')
82-
displayName: "set env. vars 4 future"
84+
displayName: "extend env. vars 4 future"
8385
8486
- bash: |
8587
echo $(DEVICES)
@@ -105,15 +107,17 @@ jobs:
105107
displayName: "Adjust dependencies"
106108
107109
- bash: |
110+
set -e
108111
extra=$(python -c "print({'lightning': 'fabric-'}.get('$(PACKAGE_NAME)', ''))")
109-
pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}" --find-links="${TORCHVISION_URL}"
112+
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
110113
pip install setuptools==75.6.0 jsonargparse==4.35.0
111114
displayName: "Install package & dependencies"
112115
113116
- bash: |
114117
set -e
115118
python requirements/collect_env_details.py
116119
python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu == 2, f'GPU: {mgpu}'"
120+
python requirements/pytorch/check-avail-extras.py
117121
python -c "import bitsandbytes"
118122
displayName: "Env details"
119123
@@ -140,10 +144,12 @@ jobs:
140144
displayName: "Testing: fabric standard"
141145
timeoutInMinutes: "10"
142146

143-
- bash: bash ./run_standalone_tests.sh "tests_fabric"
147+
- bash: |
148+
wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/run_standalone_tests.sh
149+
bash ./run_standalone_tests.sh "tests_fabric"
144150
workingDirectory: tests/
145151
env:
146-
PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
152+
PL_RUN_STANDALONE_TESTS: "1"
147153
displayName: "Testing: fabric standalone"
148154
timeoutInMinutes: "10"
149155

‎.azure/gpu-tests-pytorch.yml

+12-7
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,14 @@ jobs:
4949
cancelTimeoutInMinutes: "2"
5050
strategy:
5151
matrix:
52+
"PyTorch | oldest":
53+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.1-cuda12.1.1"
54+
PACKAGE_NAME: "pytorch"
5255
"PyTorch | latest":
53-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.11-torch2.3-cuda12.1.0"
56+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.6-cuda12.4.1"
5457
PACKAGE_NAME: "pytorch"
5558
"Lightning | latest":
56-
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.5-cuda12.1.0"
59+
image: "pytorchlightning/pytorch_lightning:base-cuda-py3.12-torch2.6-cuda12.4.1"
5760
PACKAGE_NAME: "lightning"
5861
pool: lit-rtx-3090
5962
variables:
@@ -81,9 +84,8 @@ jobs:
8184
displayName: "set env. vars"
8285
- bash: |
8386
echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/test/cu${CUDA_VERSION_MM}"
84-
echo "##vso[task.setvariable variable=TORCHVISION_URL]https://download.pytorch.org/whl/test/cu124/torchvision-0.19.0%2Bcu124-cp${PYTHON_VERSION_MM}-cp${PYTHON_VERSION_MM}-linux_x86_64.whl"
8587
condition: endsWith(variables['Agent.JobName'], 'future')
86-
displayName: "set env. vars 4 future"
88+
displayName: "extend env. vars 4 future"
8789
8890
- bash: |
8991
echo $(DEVICES)
@@ -109,8 +111,9 @@ jobs:
109111
displayName: "Adjust dependencies"
110112
111113
- bash: |
114+
set -e
112115
extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
113-
pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}" --find-links="${TORCHVISION_URL}"
116+
pip install -e ".[${extra}dev]" pytest-timeout -U --extra-index-url="${TORCH_URL}"
114117
pip install setuptools==75.6.0 jsonargparse==4.35.0
115118
displayName: "Install package & dependencies"
116119
@@ -161,11 +164,13 @@ jobs:
161164
displayName: "Testing: PyTorch standard"
162165
timeoutInMinutes: "35"
163166

164-
- bash: bash ./run_standalone_tests.sh "tests_pytorch"
167+
- bash: |
168+
wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/run_standalone_tests.sh
169+
bash ./run_standalone_tests.sh "tests_pytorch"
165170
workingDirectory: tests/
166171
env:
167172
PL_USE_MOCKED_MNIST: "1"
168-
PL_STANDALONE_TESTS_SOURCE: $(COVERAGE_SOURCE)
173+
PL_RUN_STANDALONE_TESTS: "1"
169174
displayName: "Testing: PyTorch standalone tests"
170175
timeoutInMinutes: "35"
171176

‎.github/CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ We welcome any useful contribution! For your convenience here's a recommended wo
189189
#### How can I help/contribute?
190190

191191
All types of contributions are welcome - reporting bugs, fixing documentation, adding test cases, solving issues, and preparing bug fixes.
192-
To get started with code contributions, look for issues marked with the label [good first issue](https://github.com/Lightning-AI/lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or chose something close to your domain with the label [help wanted](https://github.com/Lightning-AI/lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Before coding, make sure that the issue description is clear and comment on the issue so that we can assign it to you (or simply self-assign if you can).
192+
To get started with code contributions, look for issues marked with the label [good first issue](https://github.com/Lightning-AI/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or chose something close to your domain with the label [help wanted](https://github.com/Lightning-AI/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). Before coding, make sure that the issue description is clear and comment on the issue so that we can assign it to you (or simply self-assign if you can).
193193

194194
#### Is there a recommendation for branch names?
195195

‎.github/actions/pip-wheels/action.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ runs:
4646
run: |
4747
# cat requirements.dump
4848
pip wheel -r requirements.dump --prefer-binary \
49-
--wheel-dir=.wheels \
50-
-f ${{ inputs.torch-url }} -f ${{ inputs.wheel-dir }}
49+
--wheel-dir=".wheels" \
50+
--extra-index-url=${{ inputs.torch-url }} -f ${{ inputs.wheel-dir }}
5151
ls -lh .wheels/
5252
shell: bash
5353

‎.github/checkgroup.yml

+3-72
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,7 @@ subprojects:
1919
- "!*.md"
2020
- "!**/*.md"
2121
checks:
22-
- "pl-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
23-
- "pl-cpu (macOS-14, lightning, 3.10, 2.1)"
24-
- "pl-cpu (macOS-14, lightning, 3.11, 2.2.2)"
25-
- "pl-cpu (macOS-14, lightning, 3.11, 2.3)"
26-
- "pl-cpu (macOS-14, lightning, 3.12.7, 2.4.1)"
27-
- "pl-cpu (macOS-14, lightning, 3.12.7, 2.5.1)"
28-
- "pl-cpu (ubuntu-20.04, lightning, 3.9, 2.1, oldest)"
29-
- "pl-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
30-
- "pl-cpu (ubuntu-20.04, lightning, 3.11, 2.2.2)"
31-
- "pl-cpu (ubuntu-20.04, lightning, 3.11, 2.3)"
32-
- "pl-cpu (ubuntu-22.04, lightning, 3.12.7, 2.4.1)"
33-
- "pl-cpu (ubuntu-22.04, lightning, 3.12.7, 2.5.1)"
34-
- "pl-cpu (windows-2022, lightning, 3.9, 2.1, oldest)"
35-
- "pl-cpu (windows-2022, lightning, 3.10, 2.1)"
36-
- "pl-cpu (windows-2022, lightning, 3.11, 2.2.2)"
37-
- "pl-cpu (windows-2022, lightning, 3.11, 2.3)"
38-
- "pl-cpu (windows-2022, lightning, 3.12.7, 2.4.1)"
39-
- "pl-cpu (windows-2022, lightning, 3.12.7, 2.5.1)"
40-
- "pl-cpu (macOS-14, pytorch, 3.9, 2.1)"
41-
- "pl-cpu (ubuntu-20.04, pytorch, 3.9, 2.1)"
42-
- "pl-cpu (windows-2022, pytorch, 3.9, 2.1)"
43-
- "pl-cpu (macOS-14, pytorch, 3.12.7, 2.5.1)"
44-
- "pl-cpu (ubuntu-22.04, pytorch, 3.12.7, 2.5.1)"
45-
- "pl-cpu (windows-2022, pytorch, 3.12.7, 2.5.1)"
22+
- "pl-cpu-guardian" # aggregated check for all cases
4623

4724
- id: "pytorch_lightning: Azure GPU"
4825
paths:
@@ -172,30 +149,7 @@ subprojects:
172149
- "!*.md"
173150
- "!**/*.md"
174151
checks:
175-
- "fabric-cpu (macOS-14, lightning, 3.9, 2.1, oldest)"
176-
- "fabric-cpu (macOS-14, lightning, 3.10, 2.1)"
177-
- "fabric-cpu (macOS-14, lightning, 3.11, 2.2.2)"
178-
- "fabric-cpu (macOS-14, lightning, 3.11, 2.3)"
179-
- "fabric-cpu (macOS-14, lightning, 3.12.7, 2.4.1)"
180-
- "fabric-cpu (macOS-14, lightning, 3.12.7, 2.5.1)"
181-
- "fabric-cpu (ubuntu-20.04, lightning, 3.9, 2.1, oldest)"
182-
- "fabric-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
183-
- "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.2.2)"
184-
- "fabric-cpu (ubuntu-20.04, lightning, 3.11, 2.3)"
185-
- "fabric-cpu (ubuntu-22.04, lightning, 3.12.7, 2.4.1)"
186-
- "fabric-cpu (ubuntu-22.04, lightning, 3.12.7, 2.5.1)"
187-
- "fabric-cpu (windows-2022, lightning, 3.9, 2.1, oldest)"
188-
- "fabric-cpu (windows-2022, lightning, 3.10, 2.1)"
189-
- "fabric-cpu (windows-2022, lightning, 3.11, 2.2.2)"
190-
- "fabric-cpu (windows-2022, lightning, 3.11, 2.3)"
191-
- "fabric-cpu (windows-2022, lightning, 3.12.7, 2.4.1)"
192-
- "fabric-cpu (windows-2022, lightning, 3.12.7, 2.5.1)"
193-
- "fabric-cpu (macOS-14, fabric, 3.9, 2.1)"
194-
- "fabric-cpu (ubuntu-20.04, fabric, 3.9, 2.1)"
195-
- "fabric-cpu (windows-2022, fabric, 3.9, 2.1)"
196-
- "fabric-cpu (macOS-14, fabric, 3.12.7, 2.5.1)"
197-
- "fabric-cpu (ubuntu-22.04, fabric, 3.12.7, 2.5.1)"
198-
- "fabric-cpu (windows-2022, fabric, 3.12.7, 2.5.1)"
152+
- "fabric-cpu-guardian" # aggregated check for all cases
199153

200154
- id: "lightning_fabric: Azure GPU"
201155
paths:
@@ -259,27 +213,4 @@ subprojects:
259213
- "!*.md"
260214
- "!**/*.md"
261215
checks:
262-
- "install-pkg (ubuntu-22.04, fabric, 3.9)"
263-
- "install-pkg (ubuntu-22.04, fabric, 3.11)"
264-
- "install-pkg (ubuntu-22.04, pytorch, 3.9)"
265-
- "install-pkg (ubuntu-22.04, pytorch, 3.11)"
266-
- "install-pkg (ubuntu-22.04, lightning, 3.9)"
267-
- "install-pkg (ubuntu-22.04, lightning, 3.11)"
268-
- "install-pkg (ubuntu-22.04, notset, 3.9)"
269-
- "install-pkg (ubuntu-22.04, notset, 3.11)"
270-
- "install-pkg (macOS-14, fabric, 3.9)"
271-
- "install-pkg (macOS-14, fabric, 3.11)"
272-
- "install-pkg (macOS-14, pytorch, 3.9)"
273-
- "install-pkg (macOS-14, pytorch, 3.11)"
274-
- "install-pkg (macOS-14, lightning, 3.9)"
275-
- "install-pkg (macOS-14, lightning, 3.11)"
276-
- "install-pkg (macOS-14, notset, 3.9)"
277-
- "install-pkg (macOS-14, notset, 3.11)"
278-
- "install-pkg (windows-2022, fabric, 3.9)"
279-
- "install-pkg (windows-2022, fabric, 3.11)"
280-
- "install-pkg (windows-2022, pytorch, 3.9)"
281-
- "install-pkg (windows-2022, pytorch, 3.11)"
282-
- "install-pkg (windows-2022, lightning, 3.9)"
283-
- "install-pkg (windows-2022, lightning, 3.11)"
284-
- "install-pkg (windows-2022, notset, 3.9)"
285-
- "install-pkg (windows-2022, notset, 3.11)"
216+
- "install-pkg-guardian" # aggregated check for all cases

‎.github/workflows/_legacy-checkpoints.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ on:
4343

4444
env:
4545
LEGACY_FOLDER: "tests/legacy"
46-
TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
46+
TORCH_URL: "https://download.pytorch.org/whl/cpu/"
4747

4848
defaults:
4949
run:
@@ -67,12 +67,12 @@ jobs:
6767
PACKAGE_NAME: pytorch
6868
FREEZE_REQUIREMENTS: 1
6969
timeout-minutes: 20
70-
run: pip install . -f ${TORCH_URL}
70+
run: pip install . --extra-index-url="${TORCH_URL}"
7171
if: inputs.pl_version == ''
7272

7373
- name: Install PL version
7474
timeout-minutes: 20
75-
run: pip install "pytorch-lightning==${{ inputs.pl_version }}" -f ${TORCH_URL}
75+
run: pip install "pytorch-lightning==${{ inputs.pl_version }}" --extra-index-url="${TORCH_URL}"
7676
if: inputs.pl_version != ''
7777

7878
- name: Adjust tests -> PL

‎.github/workflows/call-clear-cache.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,18 @@ on:
2323
jobs:
2424
cron-clear:
2525
if: github.event_name == 'schedule' || github.event_name == 'pull_request'
26-
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.12.0
26+
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.14.0
2727
with:
28-
scripts-ref: v0.11.8
28+
scripts-ref: v0.14.0
2929
dry-run: ${{ github.event_name == 'pull_request' }}
3030
pattern: "latest|docs"
3131
age-days: 7
3232

3333
direct-clear:
3434
if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
35-
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.12.0
35+
uses: Lightning-AI/utilities/.github/workflows/cleanup-caches.yml@v0.14.0
3636
with:
37-
scripts-ref: v0.11.8
37+
scripts-ref: v0.14.0
3838
dry-run: ${{ github.event_name == 'pull_request' }}
3939
pattern: ${{ inputs.pattern || 'pypi_wheels' }} # setting str in case of PR / debugging
4040
age-days: ${{ fromJSON(inputs.age-days) || 0 }} # setting 0 in case of PR / debugging

‎.github/workflows/ci-check-md-links.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ on:
1414

1515
jobs:
1616
check-md-links:
17-
uses: Lightning-AI/utilities/.github/workflows/check-md-links.yml@v0.12.0
17+
uses: Lightning-AI/utilities/.github/workflows/check-md-links.yml@v0.14.0
1818
with:
1919
config-file: ".github/markdown-links-config.json"
2020
base-branch: "master"

‎.github/workflows/ci-pkg-install.yml

+14
Original file line numberDiff line numberDiff line change
@@ -103,3 +103,17 @@ jobs:
103103
LIGHTING_TESTING: 1 # path for require wrapper
104104
PY_IGNORE_IMPORTMISMATCH: 1
105105
run: python -m pytest src/lit/${PKG_NAME} --ignore-glob="**/cli/*-template/**" --doctest-plus
106+
107+
install-pkg-guardian:
108+
runs-on: ubuntu-latest
109+
needs: install-pkg
110+
if: always()
111+
steps:
112+
- run: echo "${{ needs.install-pkg.result }}"
113+
- name: failing...
114+
if: needs.install-pkg.result == 'failure'
115+
run: exit 1
116+
- name: cancelled or skipped...
117+
if: contains(fromJSON('["cancelled", "skipped"]'), needs.install-pkg.result)
118+
timeout-minutes: 1
119+
run: sleep 90

‎.github/workflows/ci-schema.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88

99
jobs:
1010
check:
11-
uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.12.0
11+
uses: Lightning-AI/utilities/.github/workflows/check-schema.yml@v0.14.0
1212
with:
1313
# skip azure due to the wrong schema file by MSFT
1414
# https://github.com/Lightning-AI/lightning-flash/pull/1455#issuecomment-1244793607

0 commit comments

Comments
 (0)