build_and_test.yml

name: Run Build and Test

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - "*"

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          make build

  e2e-tests:
    needs: build
    name: e2e / ${{ matrix.suite.name }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        suite: [
          # CLI test suites
          { group: "cmd", name: "cmd", path: "" },
          # providers suites, some of the providers are too heavy to run as single test
          { group: "pkg/providers", name: "yt", path: "yt", yt: true },
          { group: "pkg/providers", name: "providers-postgres", path: "postgres" },
          # e2e test suites
          { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
          { group: "tests/e2e", name: "pg2pg", path: "pg2pg" },
          { group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
          { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
          { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
          { group: "tests/e2e", name: "ch2s3", path: "ch2s3" },
        ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          go install gotest.tools/gotestsum@latest
      - shell: bash
        run: |
          curl https://clickhouse.com/ | sh
          sudo ./clickhouse install
      - name: Setup PostgreSQL
        uses: tj-actions/install-postgresql@v3
        with:
          postgresql-version: 16
      - shell: bash
        run: |
          pg_dump --version
      - shell: bash
        if: matrix.suite.yt
        name: prepare local YT
        run: |
          go build -o binaries/lightexe ./pkg/providers/yt/lightexe/*.go
          docker compose -f "pkg/providers/yt/recipe/docker-compose.yml" up -d --build
          export YT_PROXY=localhost:8180
          export TEST_DEPS_BINARY_PATH=binaries
      - shell: bash
        run: |
          make run-tests SUITE_GROUP="${{ matrix.suite.group }}" SUITE_PATH="${{ matrix.suite.path }}" SUITE_NAME="${{ matrix.suite.name }}"
      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-reports-${{ matrix.suite.name }}
          path: reports/*.xml
      - name: Fail if tests failed
        if: failure()
        run: exit 1

  generic-tests:
    needs: build
    name: tests - ${{ matrix.suite.name }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        suite: [
          # canon test suites
          { group: "tests/canon", name: "canon-parser", path: "parser" },
          { group: "tests/storage", name: "storage-pg", path: "pg" },
          # internal test suites
          { group: "internal", name: "internal", path: "..." },
          # provider test suites
          { group: "pkg/providers", name: "providers-mongo", path: "mongo" },
          { group: "pkg/providers", name: "providers-mysql", path: "mysql" },
          { group: "pkg/providers", name: "providers-sample", path: "sample" },
          { group: "pkg/providers", name: "providers-kafka", path: "kafka" },
          { group: "pkg/providers", name: "providers-kinesis", path: "kinesis" },
          { group: "pkg/providers", name: "providers-greenplum", path: "greenplum" },
          { group: "pkg/providers", name: "providers-clickhouse", path: "clickhouse" },
          { group: "pkg/providers", name: "providers-elastic", path: "elastic" },
          # pkg test suites
          { group: "pkg", name: "abstract", path: "abstract" },
          { group: "pkg", name: "transformer", path: "transformer" },
          { group: "pkg", name: "predicate", path: "predicate" },
          { group: "pkg", name: "dblog", path: "dblog" },
          { group: "pkg", name: "functions", path: "functions" },
          { group: "pkg", name: "maplock", path: "maplock" },
          { group: "pkg", name: "middlewares", path: "middlewares" },
          { group: "pkg", name: "parsequeue", path: "parsequeue" },
          { group: "pkg", name: "util", path: "util" },
          { group: "pkg", name: "stringutil", path: "stringutil" },
          { group: "pkg", name: "serializer", path: "serializer" },
          { group: "pkg", name: "worker", path: "worker" },
          { group: "pkg", name: "schemaregistry", path: "schemaregistry" },
          { group: "pkg", name: "parsers-generic", path: "parsers/generic" },
          { group: "pkg", name: "parsers-tests", path: "parsers/tests" },
          { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" }
        ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          go install gotest.tools/gotestsum@latest
      - shell: bash
        run: |
          curl https://clickhouse.com/ | sh
          sudo ./clickhouse install
      - name: Setup PostgreSQL
        uses: tj-actions/install-postgresql@v3
        with:
          postgresql-version: 16
      - shell: bash
        run: |
          echo "Running ${{ matrix.suite.group }} suite ${{ matrix.suite.name }}"
          export RECIPE_CLICKHOUSE_BIN=clickhouse
          export USE_TESTCONTAINERS=1
          gotestsum \
            --junitfile="reports/${{ matrix.suite.name }}.xml" \
            --junitfile-project-name="${{ matrix.suite.group }}" \
            --junitfile-testsuite-name="short" \
            --rerun-fails \
            --format github-actions \
            --packages="./${{ matrix.suite.group }}/${{ matrix.suite.path }}/..." \
            -- -timeout=15m
      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-reports-${{ matrix.suite.name }}
          path: reports/${{ matrix.suite.name }}.xml
      - name: Fail if tests failed
        if: failure()
        run: exit 1

  test-report:
    needs: [
      generic-tests,
      e2e-tests,
    ]
    name: test-report
    if: always() && !contains(needs.*.result, 'skipped')
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download All Test Reports
        uses: actions/download-artifact@v4
        with:
          pattern: test-reports-*
          merge-multiple: true
          path: reports/
      - name: Test Summary
        uses: test-summary/action@v2
        if: always()
        with:
          paths: "reports/*.xml"