diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml
index dea089ad1b503..d7c7d8cb51710 100644
--- a/.github/workflows/build-deps.yml
+++ b/.github/workflows/build-deps.yml
@@ -61,7 +61,7 @@ jobs:
       fail-fast: false
       matrix:
         job:
-        - os: arm-4core-linux
+        - os: ubuntu-22.04-arm
           image: linux-aarch64
         - os: ubuntu-22.04
           image: linux-x86_64
@@ -85,92 +85,69 @@ jobs:
 
     # On pull requests, ensure that changed files are determined before checking out the code so
     # that we use the GitHub API, otherwise we would have to fetch the entire history (depth: 0)
-    - name: Get changed files
+    - name: Check for builder changes (pull request)
+      id: changed-files-pr
+      if: github.event_name == 'pull_request'
+      env:
+        GH_TOKEN: "${{ github.token }}"
+      run: |
+        PR_NUMBER="${{ github.event.pull_request.number }}"
+        REPO="${{ github.repository }}"
+
+        BUILDERS_CHANGED=$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" | \
+          jq -r 'map(.filename) | map(select(startswith(".builders/"))) | length > 0')
+
+        echo "builders_any_changed=$BUILDERS_CHANGED" >> $GITHUB_OUTPUT
+
+    # For push events, we still need to check changes but will rely on minimal checkout
+    - name: Check for builder changes (push)
+      id: changed-files-push
+      if: github.event_name != 'pull_request'
+      run: |
+        CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD)
+        echo "builders_any_changed=$(echo "$CHANGED_FILES" | grep -q "^\.builders/" && echo "true" || echo "false")" >> $GITHUB_OUTPUT
+
+    # Combine outputs for subsequent steps
+    - name: Combine changed files outputs
       id: changed-files
-      uses: tj-actions/changed-files@v42
-      with:
-        files_yaml: |-
-          builders:
-          - .builders/**
-          dependencies:
-          - ${{ env.DIRECT_DEPENDENCY_FILE }}
+      run: |
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          echo "builders_any_changed=${{ steps.changed-files-pr.outputs.builders_any_changed }}" >> $GITHUB_OUTPUT
+        else
+          echo "builders_any_changed=${{ steps.changed-files-push.outputs.builders_any_changed }}" >> $GITHUB_OUTPUT
+        fi
 
     - name: Checkout code
       if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
 
     - name: Set up Python ${{ env.PYTHON_VERSION }}
-      if: matrix.job.image != 'linux-aarch64'
       uses: actions/setup-python@v5
       with:
         python-version: ${{ env.PYTHON_VERSION }}
 
-    - name: Set up Python (with miniconda) and other aarch64 requirements
-      if: matrix.job.image == 'linux-aarch64'
-      run: |
-        mkdir -p ~/miniconda3
-        wget https://repo.anaconda.com/miniconda/Miniconda3-py312_24.5.0-0-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
-        bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
-        rm -rf ~/miniconda3/miniconda.sh
-        ~/miniconda3/bin/conda init bash
-
-        # jq
-        wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-arm64 -O ~/miniconda3/bin/jq
-        chmod +x ~/miniconda3/bin/jq
-
-        echo "PATH=~/miniconda3/bin/:${PATH}" >> "$GITHUB_ENV"
-        echo DOCKER="sudo docker" >> "$GITHUB_ENV"
-
     - name: Install management dependencies
       run: |
         pip install -r .builders/deps/host_dependencies.txt
 
-    - name: Install docker and log in (arm64)
-      if: matrix.job.image == 'linux-aarch64'
-      run: |
-        curl -fsSL https://get.docker.com -o get-docker.sh
-        sudo sh get-docker.sh
-
-        # Logging in with sudo is necessary to get authorized with the registry when running docker under sudo
-        echo ${{ secrets.GITHUB_TOKEN }} | sudo docker login --username ${{ github.actor }} --password-stdin ghcr.io
-
     - name: Log in to GitHub Packages
-      if: matrix.job.image != 'linux-aarch64'
       uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Build image and wheels (arm64)
-      if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image == 'linux-aarch64'
-      run: |-
-        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
-        # Give ownership of the output back to the user
-        sudo chown ${USER} ${{ env.OUT_DIR }}
-
-    - name: Pull image and build wheels (arm64)
-      if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image == 'linux-aarch64'
-      run: |-
-        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
-        sudo /home/runner/miniconda3/bin/python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
-
     - name: Build image and wheels
-      if: steps.changed-files.outputs.builders_any_changed == 'true' && matrix.job.image != 'linux-aarch64'
+      if: steps.changed-files.outputs.builders_any_changed == 'true'
       run: |-
         python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
 
     - name: Pull image and build wheels
-      if: steps.changed-files.outputs.builders_any_changed != 'true' && matrix.job.image != 'linux-aarch64'
+      if: steps.changed-files.outputs.builders_any_changed != 'true'
       run: |-
         digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
         python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
 
-    - name: Change permissions
-      if: matrix.job.image == 'linux-aarch64'
-      run: |
-        sudo chmod 777 ${{ env.OUT_DIR }}
-
     - name: Publish image
       if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
       run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}