Improve the benchmark CI #66
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark branch vs thresholds | |
| on: | |
| workflow_dispatch: | |
| pull_request_review: | |
| types: [submitted] | |
| pull_request: | |
| branches: [main] | |
| types: [synchronize] | |
| paths: | |
| - "Sources/*.swift" | |
| - .github/workflows/benchmark.yml | |
| jobs: | |
| benchmark-delta-linux: | |
| # if: github.event.review.state == 'approved' | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=2cpu-linux-arm64 | |
| container: swift:jammy | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PR_COMMENT: null # will be populated later | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Configure git | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| # jemalloc is a dependency of the Benchmarking package | |
| # actions/cache will detect zstd and will become much faster. | |
| - name: Install jemalloc, curl, jq and zstd | |
| run: | | |
| set -eu | |
| apt-get update -y | |
| apt-get install -y libjemalloc-dev curl jq zstd | |
| - name: Restore .build | |
| id: "restore-cache" | |
| uses: runs-on/cache/restore@v4 | |
| with: | |
| path: Benchmarks/.build | |
| key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}" | |
| restore-keys: "swiftpm-benchmark-build-${{ runner.os }}-" | |
| - name: Run Benchmarks Of Current Branch '${{ github.head_ref || github.ref_name }}' | |
| run: | | |
| swift package -c release --disable-sandbox \ | |
| --package-path Benchmarks \ | |
| benchmark baseline update \ | |
| "${{ github.head_ref || github.ref_name }}" | |
| - name: Read Benchmark Result | |
| id: read-benchmark | |
| run: | | |
| set -eu | |
| swift package -c release --disable-sandbox \ | |
| --package-path Benchmarks \ | |
| benchmark baseline read \ | |
| "${{ github.head_ref || github.ref_name }}" \ | |
| --no-progress \ | |
| --format markdown \ | |
| >> result.text | |
| # Read the result to the output of the step | |
| echo "result<<EOF" >> $GITHUB_OUTPUT | |
| cat result.text >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Compare Current Branch '${{ github.head_ref || github.ref_name }}' Against Thresholds | |
| id: compare-benchmark | |
| run: | | |
| set -eu | |
| TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") | |
| ENCODED_TIMESTAMP=$(date -u +"%Y-%m-%dT%H%%3A%M%%3A%SZ") | |
| TIMESTAMP_LINK="https://www.timeanddate.com/worldclock/fixedtime.html?iso=$ENCODED_TIMESTAMP" | |
| echo "## Benchmark check running at [$TIMESTAMP]($TIMESTAMP_LINK)" >> summary.text | |
| # Disable 'set -e' to prevent the script from exiting on non-zero exit codes | |
| set +e | |
| swift package -c release --disable-sandbox \ | |
| --package-path Benchmarks \ | |
| benchmark thresholds check \ | |
| "${{ github.head_ref || github.ref_name }}" \ | |
| --path $PWD/Benchmarks/Thresholds/ \ | |
| --no-progress \ | |
| --format markdown \ | |
| >> summary.text | |
| echo "exit-status=$?" >> $GITHUB_OUTPUT | |
| set -e | |
| echo "summary<<EOF" >> $GITHUB_OUTPUT | |
| cat summary.text >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Cache .build | |
| if: steps.restore-cache.outputs.cache-hit != 'true' | |
| uses: runs-on/cache/save@v4 | |
| with: | |
| path: Benchmarks/.build | |
| key: "swiftpm-benchmark-build-${{ runner.os }}-${{ github.event.pull_request.base.sha || github.event.after }}" | |
| - name: Construct Comment | |
| run: | | |
| set -eu | |
| EXIT_CODE="${{ steps.compare-benchmark.outputs.exit-status }}" | |
| echo 'PR_COMMENT<<EOF' >> $GITHUB_ENV | |
| # The fact that the comment starts with <!-- benchmark ci tag. exit code: ' is used | |
| # in a other steps to find this comment again. | |
| # Be wary of that when changing this line. | |
| echo "<!-- benchmark ci tag. exit code: $EXIT_CODE -->" >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo "## [Benchmark](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) Report" >> $GITHUB_ENV | |
| case "$EXIT_CODE" in | |
| 0) | |
| echo "**✅ Pull request has no significant performance differences ✅**" >> $GITHUB_ENV | |
| ;; | |
| 1) | |
| echo "**❌ Pull request has significant performance differences 📊**" >> $GITHUB_ENV | |
| ;; | |
| 2) | |
| echo "**❌ Pull request has significant performance regressions 📉**" >> $GITHUB_ENV | |
| ;; | |
| 4) | |
| echo "**❌ Pull request has significant performance improvements 📈**" >> $GITHUB_ENV | |
| ;; | |
| *) | |
| echo "**❌ Benchmark comparison failed to complete properly with exit code $EXIT_CODE ❌**" >> $GITHUB_ENV | |
| ;; | |
| esac | |
| echo "<details>" >> $GITHUB_ENV | |
| echo " <summary> Click to expand comparison result </summary>" >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo '${{ steps.compare-benchmark.outputs.summary }}' >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo "</details>" >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo "<details>" >> $GITHUB_ENV | |
| echo " <summary> Click to expand benchmark result </summary>" >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo '${{ steps.read-benchmark.outputs.result }}' >> $GITHUB_ENV | |
| echo "" >> $GITHUB_ENV | |
| echo "</details>" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - name: Output The Comment as Job Summary | |
| run: | | |
| set -eu | |
| echo '${{ env.PR_COMMENT }}' >> $GITHUB_STEP_SUMMARY | |
| # There is a '<!-- benchmark ci tag. exit code: {some_number} -->' comment at the beginning of the benchamrk report comment. | |
| # The number in that comment is either the exit code of the last benchmark run, or a magic number that indicates the last | |
| # was in progress and did not end properly (for example if the run was cancelled). | |
| # This exit code is useful because sometimes the benchmark run is compeletely skipped because there are no changes, and | |
| # the new benchmark run that is skipped, still needs to exit with last run's exit code to keep the CI checkmark correct. | |
| # Also if the exit code is set to a magic number, we know that the last run did not complete properly and we need to | |
| # run the benchmarks regardless if this specific commit contains any benchmark changes or not, because the last run | |
| # that was in progress did indeed have come benchamrk changes otherwise it wouldn't have been run. | |
| - name: Find Existing Comment ID | |
| if: github.event_name == 'pull_request' | |
| id: existing-comment | |
| run: | | |
| set -eu | |
| # Known limitation: This only fetches the first 100 comments. This should not | |
| # matter much because a benchmark comment should have been sent early in the PR. | |
| curl -sL \ | |
| -X GET \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| -o result.json \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments?per_page=100" | |
| # Get the last comment that has a body that starts with '<!-- benchmark ci tag. exit code: '. | |
| # If available, we can just update this comment instead of creating new ones. | |
| EXISTING_COMMENT=$( | |
| cat result.json | | |
| jq ' | |
| [ | |
| .[] | | |
| select(.body | startswith("<!-- benchmark ci tag. exit code: ")) | |
| ][-1] | |
| ' | |
| ) | |
| # Check if EXISTING_COMMENT is empty or null | |
| if [ "$EXISTING_COMMENT" = "null" ] || [ -z "$EXISTING_COMMENT" ]; then | |
| ID="" | |
| BODY="" | |
| PARSED_EXIT_CODE="" | |
| else | |
| ID="$(echo "$EXISTING_COMMENT" | jq '.id')" | |
| BODY="$(echo "$EXISTING_COMMENT" | jq -r '.body')" | |
| PARSED_EXIT_CODE="$(echo "$BODY" | head -n 1 | grep -o 'exit code: [0-9]\+' | grep -o '[0-9]\+')" | |
| if [ "$ID" = "null" ]; then | |
| echo "Comment ID was null? something is wrong" | |
| echo "Comment: $EXISTING_COMMENT" | |
| exit 111 | |
| fi | |
| fi | |
| # Output the results to GITHUB_OUTPUT | |
| { | |
| echo "id=$ID" | |
| echo "exit-code=$PARSED_EXIT_CODE" | |
| echo "body<<EOF" | |
| echo "$BODY" | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Comment in PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| set -eu | |
| EXISTING_COMMENT_ID="${{ steps.existing-comment.outputs.id }}" | |
| # Need to provide the argument in a way that anything in the comment itself is not evaluated. | |
| BODY_JSON="$(jq -n -c --arg COMMENT "$(echo '${{ env.PR_COMMENT }}')" '{"body": $COMMENT}')" | |
| if [ -n "$EXISTING_COMMENT_ID" ]; then | |
| echo "Will update a comment: $EXISTING_COMMENT_ID" | |
| ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/comments/$EXISTING_COMMENT_ID" | |
| else | |
| echo "Will create a new comment" | |
| ENDPOINT="https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments" | |
| fi | |
| curl -sL \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: BEARER ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| -d "$BODY_JSON" \ | |
| "$ENDPOINT" | |
| - name: Exit With Correct Status | |
| run: | | |
| set -eu | |
| EXIT_CODE="${{ steps.existing-comment.outputs.exit-code }}" | |
| echo "Previous exit code was: $EXIT_CODE" | |
| exit $EXIT_CODE |