From f91ac27b52dd628a72005fe949d618d5ec025cdc Mon Sep 17 00:00:00 2001 From: venkkatesh-sekar <94061546+venkkatesh-sekar@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:50:00 +0100 Subject: [PATCH] feat(fuzzing): New hourly job to run fuzzers on CI (#2592) 1. Add a new script `run-all-fuzzers.sh` which runs the fuzzers for a limited amount of turns. 2. Add a hourly job `bazel-run-fuzzers-hourly` which executes each fuzzer for 100 runs. 3. Fix a regression in validator fuzzers due to changes in #1768 --------- Co-authored-by: IDX GitHub Automation --- .github/workflows-source/schedule-hourly.yml | 20 ++++++++++ .github/workflows/schedule-hourly.yml | 26 +++++++++++++ bazel/conf/.bazelrc.build | 2 +- bazel/fuzz_testing.bzl | 4 +- bin/BUILD.bazel | 10 +++++ bin/afl_wrapper.sh | 3 +- bin/run-all-fuzzers.sh | 38 +++++++++++++++++++ .../http_request_arbitrary/src/lib.rs | 4 +- 8 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 bin/BUILD.bazel create mode 100755 bin/run-all-fuzzers.sh diff --git a/.github/workflows-source/schedule-hourly.yml b/.github/workflows-source/schedule-hourly.yml index e4fc6498a880..56e26522adee 100644 --- a/.github/workflows-source/schedule-hourly.yml +++ b/.github/workflows-source/schedule-hourly.yml @@ -90,3 +90,23 @@ jobs: BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=system_test_hourly" BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }} - <<: *bazel-bep + + bazel-run-fuzzers-hourly: + name: Bazel Run Fuzzers Hourly + <<: *dind-large-setup + steps: + - <<: *checkout + - name: Run Libfuzzer targets + shell: bash + run: ./bin/run-all-fuzzers.sh --libfuzzer 100 + - name: Run AFL targets + shell: bash + run: ./bin/run-all-fuzzers.sh --afl 100 + - name: Post Slack Notification + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + if: failure() + with: + channel-id: prodsec-fuzzing + slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }} diff --git a/.github/workflows/schedule-hourly.yml b/.github/workflows/schedule-hourly.yml index 3b43b06f4a12..7ca3a663be76 100644 --- a/.github/workflows/schedule-hourly.yml +++ b/.github/workflows/schedule-hourly.yml @@ -94,3 +94,29 @@ jobs: path: | bazel-bep.pb profile.json + bazel-run-fuzzers-hourly: + name: Bazel Run Fuzzers Hourly + runs-on: + labels: dind-large + container: + image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec + options: >- + -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run Libfuzzer targets + shell: bash + run: ./bin/run-all-fuzzers.sh --libfuzzer 100 + - name: Run AFL targets + shell: bash + run: ./bin/run-all-fuzzers.sh --afl 100 + - name: Post Slack Notification + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + if: failure() + with: + channel-id: prodsec-fuzzing + slack-message: "${{ github.job }} failed :disappointed: - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Run#${{github.run_id}}>" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }} diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index 15aab99b8a63..990dd8659d9b 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -118,7 +118,7 @@ build:afl --action_env="LLVM_CONFIG=llvm-config-18" build:afl --action_env="RANLIB=llvm-ranlib-18" build:afl --config=fuzzing build:afl --build_tag_filters=afl -build:afl --run_under="/ic/bin/afl_wrapper.sh" +run:afl --run_under="//bin:afl_wrapper" # Fuzzing w/ Canister Sandbox configuration # NOTE: This is only for --config=fuzzing diff --git a/bazel/fuzz_testing.bzl b/bazel/fuzz_testing.bzl index 605383fb6e38..205dc5a3c65f 100644 --- a/bazel/fuzz_testing.bzl +++ b/bazel/fuzz_testing.bzl @@ -56,11 +56,13 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f FUZZER_LIB = [ "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a", ] + TAGS = ["sandbox_libfuzzer"] else: # default FUZZER_LIB = [ "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer-x86_64.a", ] + TAGS = [] RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + FUZZER_LIB @@ -78,7 +80,7 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f # Makes sure this target is not run in normal CI builds. It would fail due to non-nightly Rust toolchain. "fuzz_test", "libfuzzer", - ], + ] + TAGS, **kwargs ) diff --git a/bin/BUILD.bazel b/bin/BUILD.bazel new file mode 100644 index 000000000000..74815952b29e --- /dev/null +++ b/bin/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +sh_binary( + name = "afl_wrapper", + srcs = ["afl_wrapper.sh"], + tags = [ + "afl", + "fuzz_test", + ], +) diff --git a/bin/afl_wrapper.sh b/bin/afl_wrapper.sh index 06fe3dfd76be..00618a8a7a69 100755 --- a/bin/afl_wrapper.sh +++ b/bin/afl_wrapper.sh @@ -26,8 +26,6 @@ cleanup() { ps -ax | grep afl | awk '{print $1}' | xargs -I {} kill -9 {} } -trap cleanup EXIT - # This allows us to skip false positive crashes for wasm runtime if [[ "$1" == *"wasmtime"* ]] || [[ "$1" == *"wasm_executor"* ]]; then # We handle segv for wasm execution @@ -101,6 +99,7 @@ function afl_env() { # Make sure you have enough cores, as each job occupies a core. if [[ ! -z "$AFL_PARALLEL" ]]; then + trap cleanup EXIT # master fuzzer afl_env -i $INPUT_DIR -o $OUTPUT_DIR -P exploit -p explore -M fuzzer1 ${@:2} -- $1 /dev/null & diff --git a/bin/run-all-fuzzers.sh b/bin/run-all-fuzzers.sh new file mode 100755 index 000000000000..1fb2b6c670df --- /dev/null +++ b/bin/run-all-fuzzers.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# A utility script to run all fuzzers on IC for limited turns defined under MAX_EXECUTIONS +set -ex + +MAX_EXECUTIONS=100 +if [[ -n "$2" ]]; then + MAX_EXECUTIONS=$2 +fi + +case $1 in + -h | --help) + cat <&2 + usage: + $0 --libfuzzer 100 # Run all libfuzzer targets. Each target will have 100 executions. + $0 --afl 100 # Run all afl targets. Each target will have 100 executions. +EOF + exit 0 + ;; + --libfuzzer) + bazel build --config=fuzzing --build_tag_filters=libfuzzer //rs/... + LIST_OF_FUZZERS=$(bazel query 'attr(tags, "libfuzzer", //rs/...) except attr(tags, "sandbox_libfuzzer", //rs/...)') + for FUZZER in $LIST_OF_FUZZERS; do + bazel run --config=fuzzing $FUZZER -- -runs=$MAX_EXECUTIONS + done + LIST_OF_FUZZERS=$(bazel query 'attr(tags, "sandbox_libfuzzer", //rs/...)') + for FUZZER in $LIST_OF_FUZZERS; do + bazel run --config=sandbox_fuzzing $FUZZER -- -runs=$MAX_EXECUTIONS + done + ;; + + --afl) + LIST_OF_FUZZERS=$(bazel query 'attr(tags, "afl", //rs/...)') + bazel build --config=afl //rs/... + for FUZZER in $LIST_OF_FUZZERS; do + AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 bazel run --config=afl $FUZZER -- -E $MAX_EXECUTIONS + done + ;; +esac diff --git a/rs/validator/http_request_arbitrary/src/lib.rs b/rs/validator/http_request_arbitrary/src/lib.rs index 74d9189e6d18..c8b363ea0eaa 100644 --- a/rs/validator/http_request_arbitrary/src/lib.rs +++ b/rs/validator/http_request_arbitrary/src/lib.rs @@ -19,8 +19,8 @@ pub struct AnonymousContent { impl AnonymousContent { fn sender(&self) -> Blob { - const ANONYMOUS_SENDER: u8 = 0x04; - Blob(vec![ANONYMOUS_SENDER]) + const RANDOM_SENDER: [u8; 3] = [7, 1, 1]; + Blob(Vec::from(RANDOM_SENDER)) } }