From daf5bf3e118d030098f88e14ccc3b25885ee16a9 Mon Sep 17 00:00:00 2001
From: Hanno Becker <beckphan@amazon.co.uk>
Date: Tue, 25 Feb 2025 19:16:41 +0000
Subject: [PATCH] CI: Add bench build with cycles

This commit splits the `quickcheck` job in `ci.yml`
into `quickcheck` and `quickcheck_bench`, where `quickcheck`
does a simple build + test, and `quickcheck_bench` ensures
that the benchmarking binaries at least build. On MacOS, we
only test -c M1, while on Ubuntu, we test PERF and PMU.

Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>
---
 .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++++++++++++----
 scripts/tests            |  3 ++-
 2 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b0b03a93b..48aafa72f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -69,15 +69,58 @@ jobs:
       - name: tests func
         run: |
           ./scripts/tests func
-      - name: tests bench
+      - name: check namespacing
+        run: |
+          ./scripts/check-namespace
+  quickcheck_bench:
+    strategy:
+      fail-fast: false
+      matrix:
+        external:
+         - ${{ github.repository_owner != 'pq-code-package' }}
+        target:
+         - runner: pqcp-arm64
+           name: 'aarch64'
+         - runner: pqcp-arm64
+           name: 'aarch64'
+         - runner: ubuntu-latest
+           name: 'x86_64'
+         - runner: macos-latest
+           name: 'macos (aarch64)'
+         - runner: macos-13
+           name: 'macos (x86_64)'
+        exclude:
+          - {external: true,
+             target: {
+               runner: pqcp-arm64,
+               name: 'aarch64'
+             }}
+    name: Quickcheck bench (${{ matrix.target.name }})
+    runs-on: ${{ matrix.target.runner }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      # - name: "tests bench (cycles: NO)"
+      #   run: |
+      #     ./scripts/tests bench -c NO
+      - name: "tests bench (build only, cycles: PMU)"
+        if: ${{ matrix.target.name != 'macos (aarch64)' && matrix.target.name != 'macos (x86_64)' }}
         run: |
-          ./scripts/tests bench -c NO
+          make clean
+          ./scripts/tests bench -c PMU --no-run
+      - name: "tests bench (build only, cycles: PERF)"
+        if: ${{ matrix.target.name != 'macos (aarch64)' && matrix.target.name != 'macos (x86_64)' }}
+        run: |
+          make clean
+          ./scripts/tests bench -c PERF --no-run
+      - name: "tests bench (build only, cycles: M1)"
+        if: ${{ matrix.target.name == 'macos (aarch64)' || matrix.target.name == 'macos (x86_64)' }}
+        run: |
+          make clean
+          ./scripts/tests bench -c M1 --no-run
       - name: tests bench components
         run: |
+          make clean
           ./scripts/tests bench --components -c NO
-      - name: check namespacing
-        run: |
-          ./scripts/check-namespace
   quickcheck-c90:
     strategy:
       fail-fast: false
diff --git a/scripts/tests b/scripts/tests
index 89dec7a78..6dac5fc52 100755
--- a/scripts/tests
+++ b/scripts/tests
@@ -608,6 +608,7 @@ class Tests:
             output = False
 
         # NOTE: We haven't yet decided how to output both opt/no-opt benchmark results
+        resultss = None
         if self.do_opt_all():
             self._compile_schemes(test_type, False)
             if self.args.run:
@@ -623,7 +624,7 @@ class Tests:
                 )
 
         if resultss is None:
-            exit(0)
+            self.check_fail()
 
         # NOTE: There will only be one items in resultss, as we haven't yet decided how to write both opt/no-opt benchmark results
         for k, results in resultss.items():