Merge branch 'master' into bnb-update

Author: carmocca

.actions/assistant.py

@@ -44,11 +44,7 @@
         "requirements/fabric/base.txt",
         "requirements/fabric/strategies.txt",
     ),
-    "data": (
-        "requirements/data/data.txt",
-        "requirements/data/cloud.txt",
-        "requirements/data/examples.txt",
-    ),
+    "data": ("requirements/data/data.txt",),
 }
 REQUIREMENT_FILES_ALL = list(chain(*REQUIREMENT_FILES.values()))
 

.azure/gpu-tests-pytorch.yml

@@ -105,16 +105,9 @@ jobs:
           done
         displayName: "Adjust dependencies"
 
-      - bash: |
-          pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py requirements_prune_pkgs \
-            --packages="[lightning-colossalai]" \
-            --req_files="[requirements/_integrations/strategies.txt]"
-        displayName: "Prune packages" # these have installation issues
-
       - bash: |
           extra=$(python -c "print({'lightning': 'pytorch-'}.get('$(PACKAGE_NAME)', ''))")
-          pip install -e ".[${extra}dev]" -r requirements/_integrations/strategies.txt pytest-timeout -U --find-links="${TORCH_URL}"
+          pip install -e ".[${extra}dev]" pytest-timeout -U --find-links="${TORCH_URL}"
         displayName: "Install package & dependencies"
 
       - bash: pip uninstall -y lightning

.github/CODEOWNERS

@@ -41,10 +41,8 @@
 /src/lightning/pytorch/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
 /src/lightning/pytorch/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
 
-# Data Utilities
-/examples/data/           @tchaton @nohalon @justusschock @lantiga
+# Lightning Data
 /src/lightning/data/      @tchaton
-/tests/tests_data         @tchaton
 
 # Lightning Fabric
 /src/lightning/fabric                       @awaelchli @carmocca @justusschock

.github/checkgroup.yml

@@ -150,29 +150,6 @@ subprojects:
       - "build-pl (3.10, 2.0, 11.8.0)"
       - "build-pl (3.10, 2.1, 12.1.0)"
 
-  # SECTIONS: lightning_data
-
-  - id: "lightning_data: CPU workflow"
-    paths:
-      - ".actions/*"
-      - "requirements/data/**"
-      - "src/lightning/data/**"
-      - "src/lightning_data/*"
-      - "src/lightning/__init__.py"
-      - "src/lightning/__setup__.py"
-      - "src/lightning/__version__.py"
-      - "tests/tests_data/**"
-      - "examples/data/**"
-      - "pyproject.toml" # includes pytest config
-      - ".github/workflows/ci-tests-data.yml"
-      - "!requirements/*/docs.txt"
-      - "!*.md"
-      - "!**/*.md"
-    checks:
-      - "data-cpu (macOS-11, lightning, 3.10, 2.1)"
-      - "data-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "data-cpu (windows-2022, lightning, 3.10, 2.1)"
-
   # SECTION: lightning_fabric
 
   - id: "lightning_fabric: CPU workflow"

.github/label-change.yml

@@ -14,7 +14,6 @@ data:
   - changed-files:
       - any-glob-to-any-file:
           - "src/lightning/data/**"
-          - "tests/tests_data/**"
           - "requirements/data/**"
 
 store:

.github/workflows/ci-examples-app.yml

@@ -126,7 +126,7 @@ jobs:
           coverage report -i
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: tests/coverage.xml

.github/workflows/ci-pkg-install.yml

@@ -100,16 +100,22 @@ jobs:
           rm -f ./*/__*.py
           rm -f ./**/__*.py
           mv lightning lit  # rename lightning folder to prevent accidental local imports
-      - name: drop App doctest
-        if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra != 'app' }}
+      - name: drop Secondary doctest
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
         working-directory: src/lit
         run: |
-          rm -rf app
-          rm -rf data
-          rm -rf store
+          items=("data" "store" "app")
+          for item in "${items[@]}"; do
+            if [[ "$item" == "${{ matrix.pkg-extra }}" ]]; then
+              echo "Skipping $item"
+              continue # Skip this iteration
+            fi
+            echo "Removing $item"
+            rm -rf $item
+          done
       - name: Install pytest doctest extension
         run: |
-          pip install -q "pytest-doctestplus>=0.9.0"
+          pip install -q -r requirements/doctests.txt
           pip list
 
       - name: DocTest package

.github/workflows/ci-tests-app.yml

@@ -151,7 +151,7 @@ jobs:
           coverage report -i
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: tests/coverage.xml

.github/workflows/ci-tests-data.yml

@@ -173,7 +173,7 @@ jobs:
           coverage xml
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # see: https://github.com/actions/toolkit/issues/399
         continue-on-error: true
         with:

.github/workflows/ci-tests-fabric.yml

@@ -210,7 +210,7 @@ jobs:
           coverage xml
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # see: https://github.com/actions/toolkit/issues/399
         continue-on-error: true
         with:

.github/workflows/ci-tests-pytorch.yml

@@ -85,7 +85,7 @@ jobs:
           coverage xml
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         # see: https://github.com/actions/toolkit/issues/399
         continue-on-error: true
         with:

.github/workflows/ci-tests-store.yml

@@ -165,7 +165,7 @@ jobs:
           gcloud compute tpus tpu-vm list
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/tpu-tests.yml

@@ -23,6 +23,7 @@ docs/source-pytorch/_static/images/course_UvA-DL
 docs/source-pytorch/_static/images/lightning_examples
 docs/source-pytorch/_static/fetched-s3-assets
 docs/source-pytorch/integrations/hpu
+docs/source-pytorch/integrations/strategies/Hivemind.rst
 
 docs/source-fabric/*/generated
 
@@ -193,7 +194,6 @@ our_model.tar
 test.png
 saved_models
 data/
-!src/lightning/data/
 !examples/data/
 !tests/tests_pytorch/utilities/data/
 !requirements/data/

.gitignore

@@ -71,25 +71,18 @@ repos:
         additional_dependencies: [tomli]
         args: ["--in-place"]
 
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
+  - repo: https://github.com/sphinx-contrib/sphinx-lint
+    rev: v0.9.1
     hooks:
-      - id: yesqa
-        name: Unused noqa
-        additional_dependencies:
-          #- pep8-naming
-          - flake8-pytest-style
-          - flake8-bandit
-          - flake8-simplify
-          - flake8-return
+      - id: sphinx-lint
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.2.0"
     hooks:
-      - id: ruff
-        args: ["--fix", "--preview"]
       - id: ruff-format
         args: ["--preview"]
+      - id: ruff
+        args: ["--fix", "--preview"]
 
   - repo: https://github.com/executablebooks/mdformat
     rev: 0.7.17
@@ -102,7 +95,6 @@ repos:
         exclude: |
           (?x)^(
               src/lightning/app/CHANGELOG.md|
-              src/lightning/data/CHANGELOG.md|
               src/lightning/fabric/CHANGELOG.md|
               src/lightning/pytorch/CHANGELOG.md|
               README.md

.pre-commit-config.yaml

@@ -87,13 +87,13 @@ And here's the output you get when running the App using the **Lightning CLI**:
 
 .. code-block:: console
 
-	INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view
-	State: {'works': {'w': {'vars': {'counter': 1}}}}
-	State: {'works': {'w': {'vars': {'counter': 2}}}}
-	State: {'works': {'w': {'vars': {'counter': 3}}}}
-	State: {'works': {'w': {'vars': {'counter': 3}}}}
-	State: {'works': {'w': {'vars': {'counter': 4}}}}
-	...
+    INFO: Your app has started. View it in your browser: http://127.0.0.1:7501/view
+    State: {'works': {'w': {'vars': {'counter': 1}}}}
+    State: {'works': {'w': {'vars': {'counter': 2}}}}
+    State: {'works': {'w': {'vars': {'counter': 3}}}}
+    State: {'works': {'w': {'vars': {'counter': 3}}}}
+    State: {'works': {'w': {'vars': {'counter': 4}}}}
+    ...
 
 ----
 

docs/source-app/core_api/lightning_app/communication_content.rst

@@ -41,9 +41,9 @@ There are a couple of ways you can add a dynamic Work:
         def run(self):
 
             if not hasattr(self, "work"):
-		        # The `Work` component is created and attached here.
+                # The `Work` component is created and attached here.
                 setattr(self, "work", Work())
-	        # Run the `Work` component.
+            # Run the `Work` component.
             getattr(self,  "work").run()
 
 **OPTION 2:** Use the built-in Lightning classes :class:`~lightning.app.structures.Dict` or :class:`~lightning.app.structures.List`
@@ -60,7 +60,7 @@ There are a couple of ways you can add a dynamic Work:
 
         def run(self):
             if "work" not in self.dict:
-			    # The `Work` component is attached here.
+                # The `Work` component is attached here.
                 self.dict["work"] = Work()
             self.dict["work"].run()
 

docs/source-app/core_api/lightning_app/dynamic_work_content.rst

@@ -24,4 +24,4 @@ Environment variables are available in all Flows and Works, and can be accessed
     print(os.environ["BAZ"])  # FAZ
 
 .. note::
-	Environment variables are not encrypted. For sensitive values, we recommend using :ref:`Encrypted Secrets <secrets>`.
+    Environment variables are not encrypted. For sensitive values, we recommend using :ref:`Encrypted Secrets <secrets>`.

docs/source-app/glossary/environment_variables.rst

@@ -8,7 +8,7 @@ Encrypted Secrets allow you to pass private data to your apps, like API keys, ac
 Secrets provide you with a secure way to store this data in a way that is accessible to Apps so that they can authenticate third-party services/solutions.
 
 .. tip::
-	For non-sensitive configuration values, we recommend using :ref:`plain-text Environment Variables <environment_variables>`.
+    For non-sensitive configuration values, we recommend using :ref:`plain-text Environment Variables <environment_variables>`.
 
 ************
 Add a secret