Merge branch 'main' into support-ov-models-via-genai

Author: sstrehlk

.github/workflows/new_tasks.yml

@@ -16,17 +16,16 @@ jobs:
     name: Scan for changed tasks
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2  # OR "2" -> To retrieve the preceding commit.
 
       # Uses the tj-actions/changed-files action to check for changes.
-      # Outputs provided here: https://github.com/tj-actions/changed-files#outputs
       # The `files_yaml` input optionally takes a yaml string to specify filters,
       # and prepends the filter name to the standard output names.
       - name: Check task folders
         id: changed-tasks
-        uses: tj-actions/changed-files@v44.5.2
+        uses: tj-actions/changed-files@v46.0.5
         with:
           # tasks checks the tasks folder and api checks the api folder for changes
           files_yaml: |
@@ -47,7 +46,7 @@ jobs:
 
       - name: Set up Python 3.9
         if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
           cache: 'pip'

.github/workflows/publish.yml

@@ -13,10 +13,29 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.x"
 
+    - name: Check version consistency
+      run: |
+        # Extract version from pyproject.toml
+        PYPROJECT_VERSION=$(grep 'version = ' pyproject.toml | head -1 | cut -d'"' -f2)
+
+        # Extract version from __init__.py
+        INIT_VERSION=$(grep '__version__ = ' lm_eval/__init__.py | head -1 | cut -d'"' -f2)
+
+        echo "Version in pyproject.toml: $PYPROJECT_VERSION"
+        echo "Version in __init__.py: $INIT_VERSION"
+
+        # Check if versions match
+        if [ "$PYPROJECT_VERSION" != "$INIT_VERSION" ]; then
+          echo "Error: Version mismatch between pyproject.toml ($PYPROJECT_VERSION) and __init__.py ($INIT_VERSION)"
+          exit 1
+        fi
+
+        echo "Version check passed: $PYPROJECT_VERSION"
+
     - name: Install pypa/build
       run: >-
         python3 -m
@@ -26,7 +45,7 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -46,7 +65,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -68,7 +87,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/unit_tests.yml

@@ -20,76 +20,95 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.8
-        cache: pip
-        cache-dependency-path: pyproject.toml
-    - name: Pre-Commit
-      env:
-        SKIP: "no-commit-to-branch,mypy"
-
-      uses: pre-commit/action@v3.0.1
-#       # mypy turned off for now
-#    - name: Lint with mypy
-#      run: mypy . --ignore-missing-imports --check-untyped-defs --explicit-package-bases --warn-unreachable
-# Job 2
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Pre-Commit
+        env:
+          SKIP: "no-commit-to-branch,mypy"
+        uses: pre-commit/action@v3.0.1
+  # Job 2
   testcpu:
     name: CPU Tests
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: true
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: ["3.9", "3.10", "3.11"]
     timeout-minutes: 30
     steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: pip
-        cache-dependency-path: pyproject.toml
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e '.[dev,sentencepiece,api]' --extra-index-url https://download.pytorch.org/whl/cpu
-#         Install optional git dependencies
-#                pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt
-#        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Test with pytest
-      run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py
-    - name: Archive artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: output_results
-        path: |
-          test_logs/*
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      # Cache HuggingFace cache directory for CPU tests
+      - name: Cache HuggingFace cache (CPU tests)
+        uses: actions/cache@v3
+        id: cache-hf-cpu
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ runner.os }}-hf-cache-cpu
+          restore-keys: |
+            ${{ runner.os }}-hf-cache-cpu
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu
+          pip install hf_xet
+
+      - name: Test with pytest
+        run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py
+        continue-on-error: true  # Continue workflow even if tests fail
+
+      # Save test artifacts
+      - name: Archive test artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: output_testcpu${{ matrix.python-version }}
+          path: |
+            test_logs/*
+
   testmodels:
     name: External LM Tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.8
-        cache: pip
-        cache-dependency-path: pyproject.toml
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu
-    - name: Test with pytest
-      run: python -m pytest tests/models --showlocals -s -vv
-    - name: Archive artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: output_results
-        path: |
-          test_logs/*
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      # Cache HuggingFace cache directory for External LM tests
+      - name: Cache HuggingFace cache (External LM tests)
+        uses: actions/cache@v3
+        id: cache-hf-lm
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ runner.os }}-hf-cache-external-lm
+          restore-keys: |
+            ${{ runner.os }}-hf-cache-external-lm
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu
+          pip install -U transformers peft accelerate
+
+      - name: Test with pytest
+        run: python -m pytest tests/models --showlocals -s -vv
+        continue-on-error: true  # Continue workflow even if tests fail

.gitignore

@@ -8,6 +8,7 @@ build
 dist
 *.egg-info
 venv
+.venv/
 .vscode/
 temp
 __pycache__

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 exclude: ^tests/testdata/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -29,7 +29,7 @@ repos:
       - id: mixed-line-ending
         args: [--fix=lf]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.8
+    rev: v0.11.0
     hooks:
       # Run the linter.
       - id: ruff
@@ -38,14 +38,20 @@ repos:
         # Run the formatter.
       - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         exclude: >
           (?x)^(
               .*\.json|ignore.txt|lm_eval/tasks/.*|.*yaml|.*\.ipynb
           )$
         args: [--check-filenames, --check-hidden, --ignore-words=ignore.txt]
+  - repo: https://github.com/jackdewinter/pymarkdown
+    rev: v0.9.29
+    hooks:
+      - id: pymarkdown
+        exclude: ^lm_eval/tasks/
+        args: [fix, -r]
 #  - repo: https://github.com/pre-commit/mirrors-mypy
 #    rev: v1.5.1
 #    hooks:

CODEOWNERS

@@ -1 +1 @@
-* @haileyschoelkopf @lintangsutawika @baberabb
+* @baberabb @stellaathena