Resolve issue with GitHub Actions runner running out of disk space (#93)

* Reduce parallelism

* Docker pruning

* Max parallel 3 jobs

* Increase concurrent limit to 6

* New workflow for scheduled E2E tests on main

* Update .github/workflows/scheduled-e2e-tests.yaml

Co-authored-by: Oskar Hane <oh@oskarhane.com>

* Removed push rules from pr-e2e-tests.yaml

---------

Co-authored-by: Oskar Hane <oh@oskarhane.com>

Author: willtai

.github/workflows/pr-e2e-tests.yaml

@@ -15,11 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.8', '3.12']
         neo4j-version:
           - 5
         neo4j-edition:
-          - community
           - enterprise
     services:
       t2v-transformers:
@@ -50,6 +49,10 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+      - name: Docker Prune
+        run: |
+          docker system prune -af
+          docker volume prune -f
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:

.github/workflows/scheduled-e2e-tests.yaml

@@ -0,0 +1,92 @@
+name: 'Neo4j-GenAI Scheduled E2E Tests'
+
+on:
+  schedule:
+    - cron:  '0 6,9,12,15,18 * * 1-5'  # Runs every 3 hours daytime on working days
+  push:
+    branches:
+      - main
+
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 6
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        neo4j-version:
+          - 5
+        neo4j-edition:
+          - community
+          - enterprise
+    services:
+      t2v-transformers:
+        image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2-onnx
+        env:
+          ENABLE_CUDA: '0'
+      weaviate:
+        image: cr.weaviate.io/semitechnologies/weaviate:1.25.1
+        env:
+          TRANSFORMERS_INFERENCE_API: 'http://t2v-transformers:8080'
+          AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+          DEFAULT_VECTORIZER_MODULE: 'text2vec-transformers'
+          ENABLE_MODULES: 'text2vec-transformers'
+          CLUSTER_HOSTNAME: 'node1'
+        ports:
+          - 8080:8080
+          - 50051:50051
+      neo4j:
+        image: neo4j:${{ matrix.neo4j-version }}-${{ matrix.neo4j-edition }}
+        env:
+          NEO4J_AUTH: neo4j/password
+          NEO4J_ACCEPT_LICENSE_AGREEMENT: 'eval'
+          NEO4J_PLUGINS: '["apoc"]'
+        ports:
+          - 7687:7687
+          - 7474:7474
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Docker Prune
+        run: |
+          docker system prune -af
+          docker volume prune -f
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Set Python version for Poetry
+        run: poetry env use python${{ matrix.python-version }}
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      - name: Install root project
+        run: poetry install --no-interaction
+      - name: Install dependencies
+        run: poetry install --with dev
+      - name: Wait for Weaviate to start
+        shell: bash
+        run: |
+          set +e
+          count=0; until curl -s --fail localhost:8080/v1/.well-known/ready; do ((count++)); [ $count -ge 10 ] && echo "Reached maximum retry limit" && exit 1; sleep 15; done
+      - name: Run tests
+        shell: bash
+        run: |
+          if [[ "${{ matrix.neo4j-edition }}" == "community" ]]; then
+              poetry run pytest -m 'not enterprise_only' ./tests/e2e
+          else
+              poetry run pytest ./tests/e2e
+          fi