From ce3b3c6fdb678878d3d0321a42b9556e2e672ec8 Mon Sep 17 00:00:00 2001 From: bart-maykin Date: Fri, 26 Apr 2024 18:33:20 +0200 Subject: [PATCH] :construction_worker: added github workflows --- .github/workflows/ci.yml | 172 ++++++++++++++++++++++++++ .github/workflows/code_quality.yml | 121 ++++++++++++++++++ .github/workflows/codeql-analysis.yml | 62 ++++++++++ 3 files changed, 355 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/code_quality.yml create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6332839 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,172 @@ +name: ci + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + tags: + - '*' + pull_request: + workflow_dispatch: + +env: + IMAGE_NAME: maykinmedia/referentielijsten + DJANGO_SETTINGS_MODULE: referentielijsten.conf.jenkins + DB_PASSWORD: '' + DB_USER: postgres + +jobs: + # determine changed files to decide if certain jobs can be skipped or not + changed-files: + runs-on: ubuntu-latest # windows-latest | macos-latest + name: Determine changed files + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Get changed PY files + id: changed-py-files + uses: tj-actions/changed-files@v41 + with: + files: | + ^src/.+\.py + - name: Get changed JS files + id: changed-js-files + uses: tj-actions/changed-files@v41 + with: + files: | + ^src/.+\.js + - name: Get changed requirements files + id: changed-requirements + uses: tj-actions/changed-files@v41 + with: + files: ^requirements/.+\.txt$ + + outputs: + changed-py-files: ${{ steps.changed-py-files.outputs.any_changed }} + changed-js-files: ${{ steps.changed-js-files.outputs.any_changed }} + changed-requirements: ${{ steps.changed-requirements.outputs.any_changed }} + + tests: + runs-on: ubuntu-latest + # needs: + # - changed-files + + # # only run tests if source files have changed (e.g. skip for PRs that only update docs) + # if: ${{ needs.changed-files.outputs.changed-py-files == 'true'|| needs.changed-files.outputs.changed-requirements == 'true'|| github.event_name == 'push' }} + + strategy: + matrix: + postgres: ['15', '16'] + + name: Tests (PG ${{ matrix.postgres }}) + + services: + postgres: + image: postgres:${{ matrix.postgres }} + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v4 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.11' + setup-node: 'yes' + npm-ci-flags: '--legacy-peer-deps' + - name: Run tests + run: | + python src/manage.py collectstatic --noinput --link + coverage run src/manage.py test src + env: + DJANGO_SETTINGS_MODULE: referentielijsten.conf.jenkins + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' + + - name: Publish coverage report + uses: codecov/codecov-action@v1 + + docker: + needs: tests + name: Docker image build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Determine tag/commit hash + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + echo "git_hash=${GITHUB_SHA}" >> $GITHUB_OUTPUT + - name: Build the Docker image + run: | + docker build . \ + --tag $IMAGE_NAME:$RELEASE_VERSION \ + --build-arg COMMIT_HASH=${{ steps.vars.outputs.git_hash }} \ + --build-arg RELEASE=${{ steps.vars.outputs.tag }} \ + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} + + - run: docker image save -o image.tar $IMAGE_NAME:${{ steps.vars.outputs.tag }} + - name: Store image artifact + uses: actions/upload-artifact@v3 + with: + name: docker-image + path: image.tar + retention-days: 1 + + publish: + needs: + - tests + - docker + + name: Push Docker image + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.repository_owner == 'referentielijsten' # exclude PRs/forks + + steps: + - uses: actions/checkout@v4 + - name: Download built image + uses: actions/download-artifact@v3 + with: + name: docker-image + + - name: Determine tag/commit hash + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + + - name: Load image + run: | + docker image load -i image.tar + + - name: Log into registry + run: echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + + - name: Push the Docker image + run: docker push $IMAGE_NAME:${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml new file mode 100644 index 0000000..00c9485 --- /dev/null +++ b/.github/workflows/code_quality.yml @@ -0,0 +1,121 @@ +name: Code quality checks + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + paths: + - '**.py' + - '**.yml' + pull_request: + paths: + - '**.py' + - '**.yml' + workflow_dispatch: + +jobs: + isort: + name: Code imports + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run isort + run: isort --check-only --diff . + + black: + name: Code format + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run black + run: black --check --diff src docs + + flake8: + name: Code style + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run flake8 + run: flake8 src + + spdx: + name: SPDX header check + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run check + run: ./bin/check_spdx.py src + + migrations: + name: Check for model changes not present in the migrations + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:12-2.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install system packages + run: | + sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ + libgdal-dev \ + gdal-bin + + - name: Install dependencies + run: pip install -r requirements/ci.txt + + - name: Check for missing migrations + run: src/manage.py makemigrations --check --dry-run + env: + DJANGO_SETTINGS_MODULE: referentielijsten.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_NAME: postgres + DB_PASSWORD: '' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..6367046 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,62 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 23 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python', 'javascript'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2