docker-image.yml

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

name: Build windmill:main
on:
  push:
    branches: [main]
    tags: ["*"]
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "Dockerfile"

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

permissions: write-all

jobs:
  build:
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2
      - uses: depot/setup-action@v1

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Substitute EE code (EE logic is behind feature flag)
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - name: Docker meta
        id: meta-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: |
            latest=false
          tags: |
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Build and push publicly
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            features=embedding,parquet,openidconnect,jemalloc
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
            ${{ steps.meta-public.outputs.tags }}
          labels: |
            ${{ steps.meta-public.outputs.labels }}
            org.opencontainers.image.licenses=AGPLv3

  build_ee:
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2
      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
          flavor: |
            latest=false
          tags: |
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Substitute EE code
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  build_ee_312:
    runs-on: ubicloud
    if: ${{ startsWith(github.ref, 'refs/tags/') }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2
      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public-py312
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-py312
          flavor: |
            latest=false
          tags: |
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Substitute EE code
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64
          push: true
          build-args: |
            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
            PYTHON_IMAGE=python:3.12.2-slim-bookworm
          tags: |
            ${{ steps.meta-ee-public-py312.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public-py312.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  # disabled until we make it 100% reliable and add more meaningful tests
  # playwright:
  #   runs-on: [self-hosted, new]
  #   needs: [build]
  #   services:
  #     postgres:
  #       image: postgres
  #       env:
  #         POSTGRES_DB: windmill
  #         POSTGRES_USER: admin
  #         POSTGRES_PASSWORD: changeme
  #       ports:
  #         - 5432:5432
  #       options: >-
  #         --health-cmd pg_isready
  #         --health-interval 10s
  #         --health-timeout 5s
  #         --health-retries 5
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: "Docker"
  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
  #       id: docker-container
  #     - uses: actions/setup-node@v3
  #       with:
  #         node-version: 16
  #     - name: "Playwright run"
  #       timeout-minutes: 2
  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
  #     - name: "Clean up"
  #       run: docker kill ${{ steps.docker-container.outputs.id }}
  #       if: always()

  attach_amd64_binary_to_release:
    needs: [build, build_ee]
    runs-on: ubicloud
    if: ${{ startsWith(github.ref, 'refs/tags/') }}
    env:
      ARCH: amd64
    steps:
      - uses: actions/checkout@v4

      - run: |
          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev

      - run: |
          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
          # Also useful to visually check that the arch is the right opencontainers
          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev

      - uses: shrink/actions-docker-extract@v3
        id: extract
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
          path: "/usr/src/app/windmill"

      - uses: shrink/actions-docker-extract@v3
        id: extract-ee
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
          path: "/usr/src/app/windmill"

      - name: Rename binary with corresponding architecture
        run: |
          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"

      - name: Attach binary to release
        uses: softprops/action-gh-release@v1
        with:
          files: |
            ${{ steps.extract.outputs.destination }}/*
            ${{ steps.extract-ee.outputs.destination }}/*

  # attach_arm64_binary_to_release:
  #   needs: [build, build_ee]
  #   runs-on: ubicoud
  #   if: ${{ startsWith(github.ref, 'refs/tags/') }}
  #   env:
  #     ARCH: arm64
  #   steps:
  #     - uses: actions/checkout@v4

  #     - run: |
  #         # pulling docker image with desired arch so that actions-docker-extract doesn't do it
  #         docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
  #         docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev

  #     - run: |
  #         # Checks the image is in docker prior to running actions-docker-extract. It fails if not
  #         # Also useful to visually check that the arch is the right opencontainers
  #         docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
  #         docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev

  #     - uses: shrink/actions-docker-extract@v3
  #       id: extract
  #       with:
  #         image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
  #         path: "/usr/src/app/windmill"

  #     - uses: shrink/actions-docker-extract@v3
  #       id: extract-ee
  #       with:
  #         image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
  #         path: "/usr/src/app/windmill"

  #     - name: Rename binary with corresponding architecture
  #       run: |
  #         mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
  #         mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"

  #     - name: Attach binary to release
  #       uses: softprops/action-gh-release@v1
  #       with:
  #         files: |
  #           ${{ steps.extract.outputs.destination }}/*
  #           ${{ steps.extract-ee.outputs.destination }}/*

  run_integration_test:
    runs-on: ubicloud
    needs: [build_ee]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Prepare test run
        if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
        run: cd integration_tests && ./build.sh
      - name: Test run
        if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
        timeout-minutes: 15
        env:
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
        run: cd integration_tests && ./run.sh
      - name: Archive logs
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: Windmill Integration Tests Logs
          path: |
            integration_tests/logs

  tag_latest:
    runs-on: ubicloud
    needs: [run_integration_test, build]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag main and latest
        run: |
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main

  tag_latest_ee:
    runs-on: ubicloud
    needs: [run_integration_test, build_ee]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag main and latest for ee
        run: |
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main

  verify_ee_image_vulnerabilities:
    runs-on: ubicloud
    needs: [tag_latest_ee]
    # if: ${{ startsWith(github.ref, 'refs/tags/') }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Analyze for critical and high CVEs
        id: docker-scout-cves
        if: ${{ github.event_name != 'pull_request_target' }}
        uses: docker/scout-action@v1
        with:
          command: cves
          only-severities: critical,high
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
          sarif-file: sarif.output.json
          summary: true
          dockerhub-user: windmilllabs
          dockerhub-password: ${{ secrets.DOCKER_PAT }}

      - name: Upload SARIF result
        id: upload-sarif
        if: ${{ github.event_name != 'pull_request_target' }}
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: sarif.output.json

  # docker_scout_ee:
  #   runs-on: ubicloud
  #   needs: [tag_latest_ee]
  #   steps:
  #     - name: Docker Scout
  #       id: docker-scout
  #       uses: docker/scout-action@v1
  #       with:
  #         dockerhub-
  #         command: cves,recommendations,compare
  #         to-latest: true
  #         ignore-base: true
  #         ignore-unchanged: true
  #         only-fixed: true

  build_ee_nsjail:
    needs: [build_ee]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha,enable=true,priority=100,prefix=,suffix=,format=short

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          file: "./docker/DockerfileNsjail"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  build_ee_reports_privately:
    needs: [build_ee_nsjail]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          
      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha,enable=true,priority=100,prefix=,suffix=,format=short

      - name: Build and push publicly ee reports
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          file: "./docker/DockerfileReports"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  publish_ecr_s3:
    needs: [build_ee_nsjail]
    runs-on: ubicloud-standard-2-arm
    if: github.event_name != 'pull_request'
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          
      - name: get git hash
        if: github.event_name != 'pull_request'
        id: git_hash
        run: |
          git_hash=$(git rev-parse --short "$GITHUB_SHA")
          echo "GIT_HASH=${git_hash:0:7}" >> "$GITHUB_OUTPUT"

      - uses: shrink/actions-docker-extract@v3
        if: github.event_name != 'pull_request'
        id: extract
        with:
          image: |-
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail:${{ steps.git_hash.outputs.GIT_HASH }}
          path: "/static_frontend/."

      - uses: reggionick/s3-deploy@v3
        if: github.event_name != 'pull_request'
        with:
          folder: ${{ steps.extract.outputs.destination }}
          bucket: windmill-frontend
          bucket-region: us-east-1

  build_ee_cuda:
    if: ${{ startsWith(github.ref, 'refs/tags/') }}
    needs: [build_ee]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda
          flavor: |
            latest=false
          tags: |
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64
          push: true
          file: "./docker/DockerfileCuda"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  build_ee_slim:
    if: ${{ startsWith(github.ref, 'refs/tags/') }}
    needs: [build_ee]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v4
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim
          flavor: |
            latest=false
          tags: |
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64
          push: true
          file: "./docker/DockerfileSlim"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License