Set up test configuration (#792)

- Put packages for tests into requirements-tests.txt
- Sort packages in requirements.txt
- Make pyproject.toml like in other `albs-*` repos
- Ignore pytest warnings for the Pydantic package
- Create test_db and web_server_tests services in docker-compose.yml
- Added tests/test-alembic.ini and tests/test-vars.env for tests

Resolves: AlmaLinux/build-system/issues/225

.github/workflows/migrations-check.yml

@@ -30,41 +30,23 @@ jobs:
           git merge --no-ff --no-log origin/master
       - name: Create dirs and files
         run: |
-          mkdir -p /home/runner/work/albs-web-server/alts
-          mkdir -p /home/runner/work/albs-web-server/albs-frontend
-          mkdir -p /home/runner/work/albs-web-server/albs-node
-          mkdir -p /home/runner/work/albs-web-server/albs-sign-node
-          mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher
-          touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env
-          mkdir -p /home/runner/work/albs-web-server/albs-sign-file
-          touch /home/runner/work/albs-web-server/albs-sign-file/.env
-      - name: Add env file
-        run: |
-          touch ./vars.env
-          echo ALTS_TOKEN="secret" >> ./vars.env
-          echo ALMALINUX_CLIENT="secret" >> ./vars.env
-          echo ALMALINUX_CLIENT_SECRET="secret" >> ./vars.env
-          echo GITHUB_CLIENT="secret" >> ./vars.env
-          echo GITHUB_CLIENT_SECRET="secret" >> ./vars.env
-          echo POSTGRES_PASSWORD="password" >> ./vars.env
-          echo JWT_SECRET="secret" >> ./vars.env
-          echo POSTGRES_DB="almalinux-bs" >> ./vars.env
-          echo PACKAGE_BEHOLDER_ENABLED="False" >> ./vars.env
-          echo DATABASE_URL="postgresql+asyncpg://postgres:password@db/almalinux-bs" >> ./vars.env
-          echo SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/almalinux-bs" >> ./vars.env
-          echo PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/almalinux-bs" >> ./vars.env
+          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher}
+          touch ../albs-sign-file/.env
+          ln -sf tests/test-vars.env vars.env
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
+          target: web-server-tests
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Start services
-        run: docker compose up -d --no-deps web_server db
+        run: docker compose up -d test_db
       - name: Check migrations
-        run: docker compose run --rm --no-deps web_server alembic --config alws/alembic.ini upgrade head
+        run: docker compose run --rm web_server_tests alembic --config tests/test-alembic.ini upgrade head
       - name: Stop services
+        if: always()
         run: docker compose down --volumes

.github/workflows/pytest.yml

@@ -11,63 +11,39 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
-        name: Check out repository
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Check out repository
+        uses: actions/checkout@v4
+
       - name: Create dirs and files
         run: |
-          mkdir -p /home/runner/work/albs-web-server/alts
-          mkdir -p /home/runner/work/albs-web-server/albs-frontend
-          mkdir -p /home/runner/work/albs-web-server/albs-node
-          mkdir -p /home/runner/work/albs-web-server/albs-sign-file
-          mkdir -p /home/runner/work/albs-web-server/albs-sign-node
-          mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher
-          touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env
-          touch /home/runner/work/albs-web-server/albs-sign-file/.env
-      - name: Add env file
-        run: |
-          touch ./vars.env
-          echo ALTS_TOKEN="secret" >> ./vars.env
-          echo ALMALINUX_CLIENT="secret" >> ./vars.env
-          echo ALMALINUX_CLIENT_SECRET="secret" >> ./vars.env
-          echo GITHUB_CLIENT="secret" >> ./vars.env
-          echo GITHUB_CLIENT_SECRET="secret" >> ./vars.env
-          echo POSTGRES_PASSWORD="password" >> ./vars.env
-          echo JWT_SECRET="secret" >> ./vars.env
-          echo POSTGRES_DB="test-almalinux-bs" >> ./vars.env
-          echo PACKAGE_BEHOLDER_ENABLED="False" >> ./vars.env
-          echo DATABASE_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo FASTAPI_SQLA__PULP__SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_PRE_PING=True >> ./vars.env
-          echo FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_RECYCLE=3600 >> ./vars.env
-          echo FASTAPI_SQLA__ASYNC__SQLALCHEMY_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo FASTAPI_SQLA__ASYNC__SQLALCHEMY_ECHO_POOL=True >> ./vars.env
-          echo FASTAPI_SQLA__TEST__SQLALCHEMY_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env
-          echo SQLALCHEMY_POOL_PRE_PING=True >> ./vars.env
-          echo SQLALCHEMY_POOL_RECYCLE=3600 >> ./vars.env
-          cat alws/alembic.ini | sed 's/almalinux-bs/test-almalinux-bs/' | tee alws/alembic.ini
+          mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher}
+          touch ../albs-sign-file/.env
+          ln -sf tests/test-vars.env vars.env
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Build Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
+          target: web-server-tests
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
       - name: Start services
-        run: docker compose up -d --no-deps web_server db
+        run: docker compose up -d test_db
+
       - name: Run unit tests (pytest)
-        run: docker compose run -v /tmp:/tmp --no-deps web_server bash -o pipefail -c '
-          pytest -vv --ignore alws/ --cov-report term-missing:skip-covered
-          --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml --cov=alws
-          tests/ | tee /tmp/pytest-coverage.txt'
+        run: docker compose run --rm -v /tmp:/tmp web_server_tests bash -c "
+          pytest -v --cov
+            --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml
+            --cov-report term-missing:skip-covered | tee /tmp/pytest-coverage.txt"
+
       - name: Stop services
         run: docker compose down --volumes
+
       - name: Pytest coverage comment
         uses: MishaKav/pytest-coverage-comment@main
         id: coverageComment
@@ -82,6 +58,7 @@ jobs:
           hide-comment: false
           remove-link-from-badge: false
           junitxml-path: /tmp/pytest.xml
+
       - name: Create the Badge
         if: ${{ github.ref == 'refs/heads/master' && steps.coverageComment.outputs.coverage }}
         uses: schneegans/dynamic-badges-action@v1.7.0

.github/workflows/syntax.yml

@@ -18,21 +18,17 @@ jobs:
         with:
           files: |
             **/*.py
-      - name: Prepare python env
+      - name: Install python packages
         if: ${{ steps.changed-files.outputs.all_changed_files }}
-        run: |
-          python -m venv env
-          source env/bin/activate
-          pip install -U pip
-          pip install -r requirements.txt
+        run: pip install -r requirements-tests.txt
       - name: Run pylint
         id: pylint
         if: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          delimiter='$(openssl rand -hex 8)'
-          echo 'report<<$(delimiter)' >> $GITHUB_OUTPUT
-          env/bin/pylint ${{ steps.changed-files.outputs.all_changed_files }} --py-version 3.9 --exit-zero >> $GITHUB_OUTPUT
-          echo '$(delimiter)' >> $GITHUB_OUTPUT
+          delimiter=$(openssl rand -hex 8)
+          echo "report<<$delimiter" >> $GITHUB_OUTPUT
+          pylint ${{ steps.changed-files.outputs.all_changed_files }} --py-version 3.9 --exit-zero >> $GITHUB_OUTPUT
+          echo $delimiter >> $GITHUB_OUTPUT
       - name: Post pylint output
         uses: mshick/add-pr-comment@v2
         if: ${{ steps.changed-files.outputs.all_changed_files }}
@@ -48,7 +44,7 @@ jobs:
           message-id: pylint-report
       - name: Run black
         if: ${{ steps.changed-files.outputs.all_changed_files }}
-        run: env/bin/black ${{ steps.changed-files.outputs.all_changed_files }} --exclude alws/alembic --check --diff --color --target-version py39
+        run: black ${{ steps.changed-files.outputs.all_changed_files }} --exclude alws/alembic --check --diff --color --target-version py39
       - name: Run isort
         if: ${{ steps.changed-files.outputs.all_changed_files }}
-        run: env/bin/isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only --py 39
+        run: isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only --py 39

Dockerfile

@@ -1,4 +1,4 @@
-FROM almalinux/9-base:latest
+FROM almalinux/9-base:latest as web-server
 
 RUN <<EOT
   set -ex
@@ -18,3 +18,13 @@ RUN <<EOT
 EOT
 
 ADD --chmod=755 https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /
+
+
+FROM web-server as web-server-tests
+
+COPY requirements-tests.txt .
+RUN <<EOT
+  set -ex
+  pip3 install -r requirements-tests.txt
+  rm requirements-tests.txt
+EOT

docker-compose.yml

@@ -2,6 +2,24 @@ version: "3.9"
 
 services:
 
+  test_db:
+    image: postgres:13
+    env_file:
+      - vars.env
+    ports:
+      - "5432:5432"
+
+  web_server_tests:
+    image: albs-web-server-tests:latest
+    env_file:
+      - vars.env
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: web-server-tests
+    volumes:
+      - ".:/code"
+
   db:
     image: postgres:13
     env_file:
@@ -99,13 +117,13 @@ services:
     ports:
       - "8088:8000"
     build:
-      dockerfile: Dockerfile
       context: .
+      dockerfile: Dockerfile
+      target: web-server
     volumes:
       - "./alws:/code/alws"
       - "./scripts:/code/scripts"
       - "./reference_data:/code/reference_data"
-      - "./tests:/code/tests"
       - "../volumes/pulp/exports:/srv/exports"
     command: |
       bash -c "
@@ -380,7 +398,7 @@ services:
         max-file: "3"
 
   git_cacher:
-    image: quay.io/almalinuxorg/git-cacher:latest
+    image: git-cacher:latest
     build:
       dockerfile: Dockerfile.git-cacher
       context: .

pyproject.toml

@@ -3,11 +3,36 @@ profile = "black"
 split_on_trailing_comma = true
 
 [tool.black]
-line-length = 79
+line-length = 120
 skip-string-normalization = true
 # see https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html#preview-style
 preview = true
 enable-unstable-feature = ["hug_parens_with_braces_and_square_brackets"]
 
 [tool.pylint]
-max-line-length = 80
+max-line-length = 120
+
+# Minimum line length for functions/classes that require docstrings
+docstring-min-length = 50
+
+# https://pylint.readthedocs.io/en/stable/user_guide/checkers/features.html
+disable = [
+    "C0114", # missing-module-docstring
+    "R0902", # too-many-instance-attributes
+    "R0913", # too-many-arguments
+    "W1514", # unspecified-encoding
+]
+
+[tool.coverage.run]
+source = ['.']
+
+[tool.coverage.report]
+skip_empty = true
+include = ['alws/*']
+
+[tool.pytest.ini_options]
+pythonpath = '.'
+testpaths = ['tests']
+filterwarnings = [
+    'ignore:Pydantic serializer warnings:UserWarning'
+]

requirements-tests.txt

@@ -0,0 +1,9 @@
+pytest==8.1.1
+pytest-cov==5.0.0
+pyfakefs==5.4.0
+
+# Linters
+isort[colors]==5.13.2
+black==24.3.0
+pylint==3.1.0
+bandit[toml]==1.7.8

requirements.txt

@@ -1,40 +1,35 @@
-fastapi==0.110.1
-fastapi-users[all]==13.0.0
-# Can be updated only after migration to SQLAlchemy > 2.0
-fastapi-users-db-sqlalchemy==6.0.1
-uvicorn==0.29.0
-asyncpg==0.29.0
-psycopg2-binary==2.9.9
-pydantic==2.6.4
-pydantic-settings==2.2.1
+PyYAML==6.0.1
 SQLAlchemy==2.0.29
-aiohttp==3.9.3
-aiohttp-retry==2.8.3
 aiofiles==23.2.1
+aiohttp-retry==2.8.3
+aiohttp==3.9.3
 aioredis==2.0.1
-alembic==1.13.1
 alembic-postgresql-enum==1.1.2
-pytest==8.1.1
-pytest-cov==5.0.0
-isort[colors]==5.13.2
-black==24.3.0
-pylint==3.1.0
+alembic==1.13.1
+asyncpg==0.29.0
+dramatiq[rabbitmq, watch]==1.16.0
+email-validator>=2.0
+# Can be updated only after migration to SQLAlchemy > 2.0
+fastapi-users-db-sqlalchemy==6.0.1
+fastapi-users[all]==13.0.0
+fastapi==0.110.1
+feedgen==1.0.0
+httpx-oauth==0.13.3
+jinja2==3.1.3
 jmespath==1.0.1
-PyYAML==6.0.1
 lxml==5.2.1
-syncer==2.0.3
-dramatiq[rabbitmq, watch]==1.16.0
-python-multipart>=0.0.7
+markdown==3.6
+pgpy==0.6.0
 plumbum==1.8.2
+psycopg2-binary==2.9.9
+pydantic-settings==2.2.1
+pydantic==2.6.4
+python-multipart>=0.0.7
+sentry-sdk[fastapi]==1.45.0  # requires FastAPI>=0.79.0
+syncer==2.0.3
 tap.py==3.1
-jinja2==3.1.3
-pgpy==0.6.0
-markdown==3.6
-httpx-oauth==0.13.3
-feedgen==1.0.0
+uvicorn==0.29.0
 websockets==12.0
-email-validator>=2.0
-sentry-sdk[fastapi]==1.45.0  # requires FastAPI>=0.79.0
 git+https://github.com/AlmaLinux/immudb-wrapper.git@0.1.2#egg=immudb_wrapper
 git+https://github.com/AlmaLinux/errata2osv.git@0.0.3#egg=errata2osv
 git+https://github.com/AlmaLinux/albs-github-integration.git@0.3.0#egg=albs_github

tests/README.md

@@ -6,21 +6,23 @@
 
 `mock_classes.py` - a module which contain base class with `httpx` request method, setup logic for each test suite and HTTP status codes
 ## How to run tests locally
-1. Create `test-almalinux-bs` database
-2. Adjust variables in `vars.env`
+1. Adjust variables in `vars.env`
     ```
     POSTGRES_DB="test-almalinux-bs"
     POSTGRES_PASSWORD="password"
-    DATABASE_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs"
-    SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs"
-    PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs"
+    DATABASE_URL="postgresql+asyncpg://postgres:password@test_db/test-almalinux-bs"
+    SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs"
+    PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs"
     ```
-3. Up docker-compose services
+   or use `test-vars.env` in the `tests` folder
+   ```bash
+   ln -sf tests/test-vars.env vars.env
+   ```
+3. Start the `test_db` service
     ```bash
-    docker-compose up -d --no-deps web_server db
+    docker compose up -d test_db
     ```
-4. Run `pytest` within `web_server` container
+3. Run `pytest` within `web_server_tests` container
     ```bash
-    docker-compose run --no-deps --rm web_server bash -c 'source env/bin/activate && pytest -v --ignore alws/'
+    docker compose run --rm web_server_tests pytest -v
     ```
-    - we ignore `alws/` directory because it's contains files which names starts with `test*.py`