build_and_test.yml

name: Run Build and Test

on:
  push:
    branches:
      - "main"
  pull_request:
    branches:
      - "*"

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          make build

  e2e-tests:
    needs: build
    name: e2e / ${{ matrix.suite.name }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        suite: [
            # CLI test suites
            { group: "cmd", name: "cmd", path: "" },
            # providers suites, some of the providers are too heavy to run as single test
            {
              group: "pkg/providers",
              name: "container",
              path: "container",
              container: true,
            },
            { group: "pkg/providers", name: "yt", path: "yt", yt: true },
            {
              group: "pkg/providers",
              name: "providers-postgres",
              path: "postgres",
            },
            # e2e test suites
            { group: "tests/e2e", name: "kafka2ch", path: "kafka2ch" },
            { group: "tests/e2e", name: "pg2pg", path: "pg2pg" },
            { group: "tests/e2e", name: "pg2ch", path: "pg2ch" },
            { group: "tests/e2e", name: "mongo2ch", path: "mongo2ch" },
            { group: "tests/e2e", name: "kinesis2ch", path: "kinesis2ch" },
            { group: "tests/e2e", name: "ch2s3", path: "ch2s3" },
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          go install gotest.tools/gotestsum@latest
      - shell: bash
        run: |
          curl https://clickhouse.com/ | sh
          sudo ./clickhouse install
      - name: Setup PostgreSQL
        uses: tj-actions/install-postgresql@v3
        with:
          postgresql-version: 16
      - shell: bash
        run: |
          pg_dump --version
      - uses: engineerd/setup-kind@v0.6.0
        if: matrix.suite.container
        with:
          version: "v0.26.0"
          # Handled by the test code
          skipClusterCreation: true
          skipClusterDeletion: true
          skipClusterLogsExport: true
      - shell: bash
        if: matrix.suite.yt
        name: prepare local YT
        run: |
          go build -o binaries/lightexe ./pkg/providers/yt/lightexe/*.go
          docker compose -f "pkg/providers/yt/recipe/docker-compose.yml" up -d --build
          export YT_PROXY=localhost:8180
          export TEST_DEPS_BINARY_PATH=binaries
      - shell: bash
        run: |
          make run-tests SUITE_GROUP="${{ matrix.suite.group }}" SUITE_PATH="${{ matrix.suite.path }}" SUITE_NAME="${{ matrix.suite.name }}"
        env:
          TEST_KUBERNETES_INTEGRATION: ${{ matrix.suite.container == true && '1' || '' }}
      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-reports-${{ matrix.suite.name }}
          path: reports/*.xml
      - name: Fail if tests failed
        if: failure()
        run: exit 1

  generic-tests:
    needs: build
    name: tests - ${{ matrix.suite.name }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        suite: [
            # canon test suites
            { group: "tests/canon", name: "canon-parser", path: "parser" },
            { group: "tests/storage", name: "storage-pg", path: "pg" },
            # internal test suites
            { group: "internal", name: "internal", path: "..." },
            # provider test suites
            { group: "pkg/providers", name: "providers-mongo", path: "mongo" },
            { group: "pkg/providers", name: "providers-mysql", path: "mysql" },
            {
              group: "pkg/providers",
              name: "providers-sample",
              path: "sample",
            },
            { group: "pkg/providers", name: "providers-kafka", path: "kafka" },
            {
              group: "pkg/providers",
              name: "providers-kinesis",
              path: "kinesis",
            },
            {
              group: "pkg/providers",
              name: "providers-greenplum",
              path: "greenplum",
            },
            {
              group: "pkg/providers",
              name: "providers-clickhouse",
              path: "clickhouse",
            },
            {
              group: "pkg/providers",
              name: "providers-elastic",
              path: "elastic",
            },
            # pkg test suites
            { group: "pkg", name: "abstract", path: "abstract" },
            { group: "pkg", name: "transformer", path: "transformer" },
            { group: "pkg", name: "predicate", path: "predicate" },
            { group: "pkg", name: "dblog", path: "dblog" },
            { group: "pkg", name: "functions", path: "functions" },
            { group: "pkg", name: "maplock", path: "maplock" },
            { group: "pkg", name: "middlewares", path: "middlewares" },
            { group: "pkg", name: "parsequeue", path: "parsequeue" },
            { group: "pkg", name: "util", path: "util" },
            { group: "pkg", name: "stringutil", path: "stringutil" },
            { group: "pkg", name: "serializer", path: "serializer" },
            { group: "pkg", name: "worker", path: "worker" },
            { group: "pkg", name: "schemaregistry", path: "schemaregistry" },
            { group: "pkg", name: "parsers-generic", path: "parsers/generic" },
            { group: "pkg", name: "parsers-tests", path: "parsers/tests" },
            { group: "pkg", name: "parsers-scanner", path: "parsers/scanner" },
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22.0"
      - shell: bash
        run: |
          go install gotest.tools/gotestsum@latest
      - shell: bash
        run: |
          curl https://clickhouse.com/ | sh
          sudo ./clickhouse install
      - name: Setup PostgreSQL
        uses: tj-actions/install-postgresql@v3
        with:
          postgresql-version: 16
      - shell: bash
        run: |
          echo "Running ${{ matrix.suite.group }} suite ${{ matrix.suite.name }}"
          export RECIPE_CLICKHOUSE_BIN=clickhouse
          export USE_TESTCONTAINERS=1
          gotestsum \
            --junitfile="reports/${{ matrix.suite.name }}.xml" \
            --junitfile-project-name="${{ matrix.suite.group }}" \
            --junitfile-testsuite-name="short" \
            --rerun-fails \
            --format github-actions \
            --packages="./${{ matrix.suite.group }}/${{ matrix.suite.path }}/..." \
            -- -timeout=15m
      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-reports-${{ matrix.suite.name }}
          path: reports/${{ matrix.suite.name }}.xml
      - name: Fail if tests failed
        if: failure()
        run: exit 1

  test-report:
    needs: [generic-tests, e2e-tests]
    name: test-report
    if: always() && !contains(needs.*.result, 'skipped')
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download All Test Reports
        uses: actions/download-artifact@v4
        with:
          pattern: test-reports-*
          merge-multiple: true
          path: reports/
      - name: Test Summary
        uses: test-summary/action@v2
        if: always()
        with:
          paths: "reports/*.xml"