job.bench.yml

name: Benchmark

"on":
  workflow_dispatch: {}
  workflow_call: {}

env:
  BUILDLESS_APIKEY: ${{ secrets.BUILDLESS_APIKEY }}

permissions:
  contents: read

jobs:
  ##
  ## Job: Benchmarks
  ##

  bench:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu]
        java: [21]
        engine: [graalvm]
        experimental: [false]
        include:
          - os: macos
            java: 21
            engine: graalvm
            experimental: false

    name: "JVM"
    runs-on: ${{ matrix.os }}-latest
    continue-on-error: ${{ matrix.experimental }}

    if: |
      (
        github.ref == 'refs/heads/stable' ||
        github.ref == 'refs/heads/v3' ||
        startsWith(github.ref, 'refs/tags/v') ||
        contains(github.event.pull_request.labels.*.name, 'ci:bench') ||
        contains(github.event.head_commit.message, 'ci:bench')
      )

    permissions:
      actions: "read"
      checks: "write"
      contents: "write"
      pull-requests: "write"

    defaults:
      run:
        shell: bash

    steps:
      - name: "Setup: Harden Runner"
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit
      - name: "Setup: Checkout"
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          persist-credentials: false
          fetch-depth: 0
      - name: "Setup: Cache Restore"
        id: cache-restore
        uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
        with:
          key: elide-framework-v1-build-${{ hashFiles('gradle/elide.versions.toml') }}
          path: "**/*.*"
          restore-keys: |
            elide-framework-v1-build-${{ hashFiles('gradle/elide.versions.toml') }}
            elide-framework-v1-build-
            elide-framework-v1-
            elide-framework-
      - name: "Setup: GraalVM"
        uses: graalvm/setup-graalvm@2a93b69fdf86ac5a078a98c1a707744632e1da94 # v1.1.5
        if: ${{ matrix.engine == 'graalvm' }}
        with:
          distribution: "graalvm"
          java-version: ${{ matrix.java }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: "Setup: Zulu"
        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0
        if: ${{ matrix.engine != 'graalvm' }}
        with:
          distribution: ${{ matrix.engine }}
          java-version: ${{ matrix.java }}
      - name: "Setup: Node"
        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
        with:
          node-version: 20
      - name: "Setup: Yarn"
        run: yarn
      - name: "Run Benchmarks"
        uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0
        id: gradlebench
        continue-on-error: ${{ matrix.experimental }}
        env:
          CI: true
        with:
          cache-read-only: true
          cache-encryption-key: ${{ secrets.GRADLE_CONFIGURATION_KEY }}
          arguments: |
            :benchmarks:core:benchmark
            :benchmarks:graalvm:benchmark
            :benchmarks:server:benchmark
            --no-daemon
            --warning-mode=none
            --dependency-verification=lenient
            -Pelide.ci=true
            -PbuildBenchmarks=true
            -PbuildSamples=false
            -PbuildDocs=false
            -Pversions.java.language=${{ matrix.java }}
      - name: "Locate Benchmark Report"
        id: benchreport
        run: |
          echo "BENCHMARK_REPORT=$(find benchmarks/server/build/reports/benchmarks/main -name "*.json" | head -n 1)" >> $GITHUB_OUTPUT
      - name: "Restore Previous Benchmark"
        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
        with:
          path: ./benchmark_reports
          key: ${{ runner.os }}-${{ matrix.engine }}-${{ matrix.java }}-benchmark
      - name: "Upload Benchmark Report"
        uses: benchmark-action/github-action-benchmark@70405016b032d44f409e4b1b451c40215cbe2393 # v1.18.0
        continue-on-error: true
        with:
          name: "Elide: Server (${{ matrix.os }}, ${{ matrix.engine }}, JDK ${{ matrix.java }})"
          tool: "jmh"
          auto-push: false
          external-data-json-path: benchmark_reports/benchmark-data.json
          comment-on-alert: true
          comment-always: ${{ !matrix.experimental }}
          output-file-path: ${{ steps.benchreport.outputs.BENCHMARK_REPORT }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          fail-on-alert: ${{ !matrix.experimental }}