feat(fuzzing): New hourly job to run fuzzers on CI (#2592)

1. Add a new script `run-all-fuzzers.sh` which runs the fuzzers for a
limited amount of turns.
2. Add a hourly job `bazel-run-fuzzers-hourly` which executes each
fuzzer for 100 runs.
3. Fix a regression in validator fuzzers due to changes in #1768

---------

Co-authored-by: IDX GitHub Automation <IDX GitHub Automation>

.github/workflows-source/schedule-hourly.yml

@@ -90,3 +90,23 @@ jobs:
           BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=system_test_hourly"
           BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
       - <<: *bazel-bep
+
+  bazel-run-fuzzers-hourly:
+    name: Bazel Run Fuzzers Hourly
+    <<: *dind-large-setup
+    steps:
+      - <<: *checkout
+      - name: Run Libfuzzer targets
+        shell: bash
+        run: ./bin/run-all-fuzzers.sh --libfuzzer 100
+      - name: Run AFL targets
+        shell: bash
+        run: ./bin/run-all-fuzzers.sh --afl 100
+      - name: Post Slack Notification
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: failure()
+        with:
+          channel-id: prodsec-fuzzing
+          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}

.github/workflows/schedule-hourly.yml

@@ -94,3 +94,29 @@ jobs:
           path: |
             bazel-bep.pb
             profile.json
+  bazel-run-fuzzers-hourly:
+    name: Bazel Run Fuzzers Hourly
+    runs-on:
+      labels: dind-large
+    container:
+      image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
+      options: >-
+        -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
+    timeout-minutes: 120
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run Libfuzzer targets
+        shell: bash
+        run: ./bin/run-all-fuzzers.sh --libfuzzer 100
+      - name: Run AFL targets
+        shell: bash
+        run: ./bin/run-all-fuzzers.sh --afl 100
+      - name: Post Slack Notification
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: failure()
+        with:
+          channel-id: prodsec-fuzzing
+          slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}

bazel/conf/.bazelrc.build

@@ -118,7 +118,7 @@ build:afl --action_env="LLVM_CONFIG=llvm-config-18"
 build:afl --action_env="RANLIB=llvm-ranlib-18"
 build:afl --config=fuzzing
 build:afl --build_tag_filters=afl
-build:afl --run_under="/ic/bin/afl_wrapper.sh"
+run:afl --run_under="//bin:afl_wrapper"
 
 # Fuzzing w/ Canister Sandbox configuration
 # NOTE: This is only for --config=fuzzing

bazel/fuzz_testing.bzl

@@ -56,11 +56,13 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f
         FUZZER_LIB = [
             "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a",
         ]
+        TAGS = ["sandbox_libfuzzer"]
     else:
         # default
         FUZZER_LIB = [
             "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer-x86_64.a",
         ]
+        TAGS = []
 
     RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + FUZZER_LIB
 
@@ -78,7 +80,7 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f
             # Makes sure this target is not run in normal CI builds. It would fail due to non-nightly Rust toolchain.
             "fuzz_test",
             "libfuzzer",
-        ],
+        ] + TAGS,
         **kwargs
     )
 

bin/BUILD.bazel

@@ -0,0 +1,10 @@
+package(default_visibility = ["//visibility:public"])
+
+sh_binary(
+    name = "afl_wrapper",
+    srcs = ["afl_wrapper.sh"],
+    tags = [
+        "afl",
+        "fuzz_test",
+    ],
+)

bin/afl_wrapper.sh

@@ -26,8 +26,6 @@ cleanup() {
     ps -ax | grep afl | awk '{print $1}' | xargs -I {} kill -9 {}
 }
 
-trap cleanup EXIT
-
 # This allows us to skip false positive crashes for wasm runtime
 if [[ "$1" == *"wasmtime"* ]] || [[ "$1" == *"wasm_executor"* ]]; then
     # We handle segv for wasm execution
@@ -101,6 +99,7 @@ function afl_env() {
 # Make sure you have enough cores, as each job occupies a core.
 
 if [[ ! -z "$AFL_PARALLEL" ]]; then
+    trap cleanup EXIT
     # master fuzzer
     afl_env -i $INPUT_DIR -o $OUTPUT_DIR -P exploit -p explore -M fuzzer1 ${@:2} -- $1 </dev/null &>/dev/null &
 

bin/run-all-fuzzers.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# A utility script to run all fuzzers on IC for limited turns defined under MAX_EXECUTIONS
+set -ex
+
+MAX_EXECUTIONS=100
+if [[ -n "$2" ]]; then
+    MAX_EXECUTIONS=$2
+fi
+
+case $1 in
+    -h | --help)
+        cat <<EOF >&2
+        usage:
+            $0 --libfuzzer 100 # Run all libfuzzer targets. Each target will have 100 executions.
+            $0 --afl 100       # Run all afl targets. Each target will have 100 executions.
+EOF
+        exit 0
+        ;;
+    --libfuzzer)
+        bazel build --config=fuzzing --build_tag_filters=libfuzzer //rs/...
+        LIST_OF_FUZZERS=$(bazel query 'attr(tags, "libfuzzer", //rs/...) except attr(tags, "sandbox_libfuzzer", //rs/...)')
+        for FUZZER in $LIST_OF_FUZZERS; do
+            bazel run --config=fuzzing $FUZZER -- -runs=$MAX_EXECUTIONS
+        done
+        LIST_OF_FUZZERS=$(bazel query 'attr(tags, "sandbox_libfuzzer", //rs/...)')
+        for FUZZER in $LIST_OF_FUZZERS; do
+            bazel run --config=sandbox_fuzzing $FUZZER -- -runs=$MAX_EXECUTIONS
+        done
+        ;;
+
+    --afl)
+        LIST_OF_FUZZERS=$(bazel query 'attr(tags, "afl", //rs/...)')
+        bazel build --config=afl //rs/...
+        for FUZZER in $LIST_OF_FUZZERS; do
+            AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 bazel run --config=afl $FUZZER -- -E $MAX_EXECUTIONS
+        done
+        ;;
+esac

rs/validator/http_request_arbitrary/src/lib.rs

@@ -19,8 +19,8 @@ pub struct AnonymousContent {
 
 impl AnonymousContent {
     fn sender(&self) -> Blob {
-        const ANONYMOUS_SENDER: u8 = 0x04;
-        Blob(vec![ANONYMOUS_SENDER])
+        const RANDOM_SENDER: [u8; 3] = [7, 1, 1];
+        Blob(Vec::from(RANDOM_SENDER))
     }
 }