diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 6499f0a0e..cf232bcdb 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -30,41 +30,23 @@ jobs: git merge --no-ff --no-log origin/master - name: Create dirs and files run: | - mkdir -p /home/runner/work/albs-web-server/alts - mkdir -p /home/runner/work/albs-web-server/albs-frontend - mkdir -p /home/runner/work/albs-web-server/albs-node - mkdir -p /home/runner/work/albs-web-server/albs-sign-node - mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher - touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env - mkdir -p /home/runner/work/albs-web-server/albs-sign-file - touch /home/runner/work/albs-web-server/albs-sign-file/.env - - name: Add env file - run: | - touch ./vars.env - echo ALTS_TOKEN="secret" >> ./vars.env - echo ALMALINUX_CLIENT="secret" >> ./vars.env - echo ALMALINUX_CLIENT_SECRET="secret" >> ./vars.env - echo GITHUB_CLIENT="secret" >> ./vars.env - echo GITHUB_CLIENT_SECRET="secret" >> ./vars.env - echo POSTGRES_PASSWORD="password" >> ./vars.env - echo JWT_SECRET="secret" >> ./vars.env - echo POSTGRES_DB="almalinux-bs" >> ./vars.env - echo PACKAGE_BEHOLDER_ENABLED="False" >> ./vars.env - echo DATABASE_URL="postgresql+asyncpg://postgres:password@db/almalinux-bs" >> ./vars.env - echo SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/almalinux-bs" >> ./vars.env - echo PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/almalinux-bs" >> ./vars.env + mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher} + touch ../albs-sign-file/.env + ln -sf tests/test-vars.env vars.env - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build Docker image uses: docker/build-push-action@v5 with: context: . + target: web-server-tests load: true cache-from: type=gha cache-to: type=gha,mode=max - name: Start services - run: docker compose up -d --no-deps web_server db + run: docker compose up -d test_db - name: Check migrations - run: docker compose run --rm --no-deps web_server alembic --config alws/alembic.ini upgrade head + run: docker compose run --rm web_server_tests alembic --config tests/test-alembic.ini upgrade head - name: Stop services + if: always() run: docker compose down --volumes diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index cf33aa1d0..25bded783 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -11,63 +11,39 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - name: Check out repository - with: - ref: ${{ github.event.pull_request.head.sha }} + - name: Check out repository + uses: actions/checkout@v4 + - name: Create dirs and files run: | - mkdir -p /home/runner/work/albs-web-server/alts - mkdir -p /home/runner/work/albs-web-server/albs-frontend - mkdir -p /home/runner/work/albs-web-server/albs-node - mkdir -p /home/runner/work/albs-web-server/albs-sign-file - mkdir -p /home/runner/work/albs-web-server/albs-sign-node - mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher - touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env - touch /home/runner/work/albs-web-server/albs-sign-file/.env - - name: Add env file - run: | - touch ./vars.env - echo ALTS_TOKEN="secret" >> ./vars.env - echo ALMALINUX_CLIENT="secret" >> ./vars.env - echo ALMALINUX_CLIENT_SECRET="secret" >> ./vars.env - echo GITHUB_CLIENT="secret" >> ./vars.env - echo GITHUB_CLIENT_SECRET="secret" >> ./vars.env - echo POSTGRES_PASSWORD="password" >> ./vars.env - echo JWT_SECRET="secret" >> ./vars.env - echo POSTGRES_DB="test-almalinux-bs" >> ./vars.env - echo PACKAGE_BEHOLDER_ENABLED="False" >> ./vars.env - echo DATABASE_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo FASTAPI_SQLA__PULP__SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_PRE_PING=True >> ./vars.env - echo FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_RECYCLE=3600 >> ./vars.env - echo FASTAPI_SQLA__ASYNC__SQLALCHEMY_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo FASTAPI_SQLA__ASYNC__SQLALCHEMY_ECHO_POOL=True >> ./vars.env - echo FASTAPI_SQLA__TEST__SQLALCHEMY_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" >> ./vars.env - echo SQLALCHEMY_POOL_PRE_PING=True >> ./vars.env - echo SQLALCHEMY_POOL_RECYCLE=3600 >> ./vars.env - cat alws/alembic.ini | sed 's/almalinux-bs/test-almalinux-bs/' | tee alws/alembic.ini + mkdir -p ../{alts,albs-frontend,albs-node,albs-sign-file,albs-sign-node,alma-tests-cacher} + touch ../albs-sign-file/.env + ln -sf tests/test-vars.env vars.env + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build Docker image uses: docker/build-push-action@v5 with: context: . + target: web-server-tests load: true cache-from: type=gha cache-to: type=gha,mode=max + - name: Start services - run: docker compose up -d --no-deps web_server db + run: docker compose up -d test_db + - name: Run unit tests (pytest) - run: docker compose run -v /tmp:/tmp --no-deps web_server bash -o pipefail -c ' - pytest -vv --ignore alws/ --cov-report term-missing:skip-covered - --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml --cov=alws - tests/ | tee /tmp/pytest-coverage.txt' + run: docker compose run --rm -v /tmp:/tmp web_server_tests bash -c " + pytest -v --cov + --cov-report xml:/tmp/coverage.xml --junitxml=/tmp/pytest.xml + --cov-report term-missing:skip-covered | tee /tmp/pytest-coverage.txt" + - name: Stop services run: docker compose down --volumes + - name: Pytest coverage comment uses: MishaKav/pytest-coverage-comment@main id: coverageComment @@ -82,6 +58,7 @@ jobs: hide-comment: false remove-link-from-badge: false junitxml-path: /tmp/pytest.xml + - name: Create the Badge if: ${{ github.ref == 'refs/heads/master' && steps.coverageComment.outputs.coverage }} uses: schneegans/dynamic-badges-action@v1.7.0 diff --git a/.github/workflows/syntax.yml b/.github/workflows/syntax.yml index ab261c35c..21f7384b0 100644 --- a/.github/workflows/syntax.yml +++ b/.github/workflows/syntax.yml @@ -18,21 +18,17 @@ jobs: with: files: | **/*.py - - name: Prepare python env + - name: Install python packages if: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - python -m venv env - source env/bin/activate - pip install -U pip - pip install -r requirements.txt + run: pip install -r requirements-tests.txt - name: Run pylint id: pylint if: ${{ steps.changed-files.outputs.all_changed_files }} run: | - delimiter='$(openssl rand -hex 8)' - echo 'report<<$(delimiter)' >> $GITHUB_OUTPUT - env/bin/pylint ${{ steps.changed-files.outputs.all_changed_files }} --py-version 3.9 --exit-zero >> $GITHUB_OUTPUT - echo '$(delimiter)' >> $GITHUB_OUTPUT + delimiter=$(openssl rand -hex 8) + echo "report<<$delimiter" >> $GITHUB_OUTPUT + pylint ${{ steps.changed-files.outputs.all_changed_files }} --py-version 3.9 --exit-zero >> $GITHUB_OUTPUT + echo $delimiter >> $GITHUB_OUTPUT - name: Post pylint output uses: mshick/add-pr-comment@v2 if: ${{ steps.changed-files.outputs.all_changed_files }} @@ -48,7 +44,7 @@ jobs: message-id: pylint-report - name: Run black if: ${{ steps.changed-files.outputs.all_changed_files }} - run: env/bin/black ${{ steps.changed-files.outputs.all_changed_files }} --exclude alws/alembic --check --diff --color --target-version py39 + run: black ${{ steps.changed-files.outputs.all_changed_files }} --exclude alws/alembic --check --diff --color --target-version py39 - name: Run isort if: ${{ steps.changed-files.outputs.all_changed_files }} - run: env/bin/isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only --py 39 + run: isort ${{ steps.changed-files.outputs.all_changed_files }} --diff --color --check-only --py 39 diff --git a/Dockerfile b/Dockerfile index a56fb2cbe..a1d72e362 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM almalinux/9-base:latest +FROM almalinux/9-base:latest as web-server RUN < 2.0 -fastapi-users-db-sqlalchemy==6.0.1 -uvicorn==0.29.0 -asyncpg==0.29.0 -psycopg2-binary==2.9.9 -pydantic==2.6.4 -pydantic-settings==2.2.1 +PyYAML==6.0.1 SQLAlchemy==2.0.29 -aiohttp==3.9.3 -aiohttp-retry==2.8.3 aiofiles==23.2.1 +aiohttp-retry==2.8.3 +aiohttp==3.9.3 aioredis==2.0.1 -alembic==1.13.1 alembic-postgresql-enum==1.1.2 -pytest==8.1.1 -pytest-cov==5.0.0 -isort[colors]==5.13.2 -black==24.3.0 -pylint==3.1.0 +alembic==1.13.1 +asyncpg==0.29.0 +dramatiq[rabbitmq, watch]==1.16.0 +email-validator>=2.0 +# Can be updated only after migration to SQLAlchemy > 2.0 +fastapi-users-db-sqlalchemy==6.0.1 +fastapi-users[all]==13.0.0 +fastapi==0.110.1 +feedgen==1.0.0 +httpx-oauth==0.13.3 +jinja2==3.1.3 jmespath==1.0.1 -PyYAML==6.0.1 lxml==5.2.1 -syncer==2.0.3 -dramatiq[rabbitmq, watch]==1.16.0 -python-multipart>=0.0.7 +markdown==3.6 +pgpy==0.6.0 plumbum==1.8.2 +psycopg2-binary==2.9.9 +pydantic-settings==2.2.1 +pydantic==2.6.4 +python-multipart>=0.0.7 +sentry-sdk[fastapi]==1.45.0 # requires FastAPI>=0.79.0 +syncer==2.0.3 tap.py==3.1 -jinja2==3.1.3 -pgpy==0.6.0 -markdown==3.6 -httpx-oauth==0.13.3 -feedgen==1.0.0 +uvicorn==0.29.0 websockets==12.0 -email-validator>=2.0 -sentry-sdk[fastapi]==1.45.0 # requires FastAPI>=0.79.0 git+https://github.com/AlmaLinux/immudb-wrapper.git@0.1.2#egg=immudb_wrapper git+https://github.com/AlmaLinux/errata2osv.git@0.0.3#egg=errata2osv git+https://github.com/AlmaLinux/albs-github-integration.git@0.3.0#egg=albs_github diff --git a/tests/README.md b/tests/README.md index 90a1e45d1..7751760f2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,21 +6,23 @@ `mock_classes.py` - a module which contain base class with `httpx` request method, setup logic for each test suite and HTTP status codes ## How to run tests locally -1. Create `test-almalinux-bs` database -2. Adjust variables in `vars.env` +1. Adjust variables in `vars.env` ``` POSTGRES_DB="test-almalinux-bs" POSTGRES_PASSWORD="password" - DATABASE_URL="postgresql+asyncpg://postgres:password@db/test-almalinux-bs" - SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" - PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@db/test-almalinux-bs" + DATABASE_URL="postgresql+asyncpg://postgres:password@test_db/test-almalinux-bs" + SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" + PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" ``` -3. Up docker-compose services + or use `test-vars.env` in the `tests` folder + ```bash + ln -sf tests/test-vars.env vars.env + ``` +3. Start the `test_db` service ```bash - docker-compose up -d --no-deps web_server db + docker compose up -d test_db ``` -4. Run `pytest` within `web_server` container +3. Run `pytest` within `web_server_tests` container ```bash - docker-compose run --no-deps --rm web_server bash -c 'source env/bin/activate && pytest -v --ignore alws/' + docker compose run --rm web_server_tests pytest -v ``` - - we ignore `alws/` directory because it's contains files which names starts with `test*.py` diff --git a/tests/fixtures/database.py b/tests/fixtures/database.py index f61acb948..0ac99627b 100644 --- a/tests/fixtures/database.py +++ b/tests/fixtures/database.py @@ -1,3 +1,4 @@ +import os import typing from contextlib import asynccontextmanager @@ -12,7 +13,7 @@ from tests.constants import ADMIN_USER_ID, CUSTOM_USER_ID engine = create_async_engine( - settings.test_database_url, + os.getenv('DATABASE_URL', settings.test_database_url), poolclass=NullPool, echo_pool=True, ) diff --git a/tests/test-alembic.ini b/tests/test-alembic.ini new file mode 100644 index 000000000..63753763a --- /dev/null +++ b/tests/test-alembic.ini @@ -0,0 +1,39 @@ +[alembic] +script_location = alws/alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/tests/test-vars.env b/tests/test-vars.env new file mode 100644 index 000000000..ab3a0dc53 --- /dev/null +++ b/tests/test-vars.env @@ -0,0 +1,20 @@ +ALMALINUX_CLIENT="secret" +ALMALINUX_CLIENT_SECRET="secret" +ALTS_TOKEN="secret" +DATABASE_URL="postgresql+asyncpg://postgres:password@test_db/test-almalinux-bs" +FASTAPI_SQLA__ASYNC__SQLALCHEMY_ECHO_POOL=True +FASTAPI_SQLA__ASYNC__SQLALCHEMY_URL="postgresql+asyncpg://postgres:password@test_db/test-almalinux-bs" +FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_PRE_PING=True +FASTAPI_SQLA__PULP__SQLALCHEMY_POOL_RECYCLE=3600 +FASTAPI_SQLA__PULP__SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" +GITHUB_CLIENT="secret" +GITHUB_CLIENT_SECRET="secret" +JWT_SECRET="secret" +PACKAGE_BEHOLDER_ENABLED="False" +POSTGRES_DB="test-almalinux-bs" +POSTGRES_PASSWORD="password" +PULP_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" +SQLALCHEMY_POOL_PRE_PING=True +SQLALCHEMY_POOL_RECYCLE=3600 +SQLALCHEMY_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" +SYNC_DATABASE_URL="postgresql+psycopg2://postgres:password@test_db/test-almalinux-bs" \ No newline at end of file