From 49658964a391c37f71b1328013e9b083bb0692bb Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Mon, 18 Nov 2024 19:18:08 -0800 Subject: [PATCH] Include preliminary job from separate YAML file --- .azure-pipelines.yml | 1 + .github/workflows/clang_sanitizers.yml | 306 +++++++++++-------------- .github/workflows/skip_checks.yml | 17 ++ .pre-commit-config.yaml | 2 +- 4 files changed, 154 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/skip_checks.yml diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 72daa035fba..b3337a6840b 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -15,6 +15,7 @@ pr: - Docs - .github/workflows/** - .azure-pipelines.yml + - .pre-commit-config.yaml jobs: - job: diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml index a5610fcfb94..f1964403c36 100644 --- a/.github/workflows/clang_sanitizers.yml +++ b/.github/workflows/clang_sanitizers.yml @@ -11,176 +11,140 @@ concurrency: cancel-in-progress: true jobs: - - skip_checks: - name: Skip checks? - runs-on: macos-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 + - skip_checks: + uses: .github/workflows/skip_checks.yml + - build_UB_sanitizer: + name: Clang UB sanitizer + runs-on: ubuntu-22.04 + container: ubuntu:23.10 + needs: skip_checks + if: ${{ github.event.pull_request.draft == false && needs.skip_checks.outputs.skip == false }} + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang17.sh + - name: CCache Cache + uses: actions/cache@v4 with: - fetch-depth: 0 - - name: Run PR analysis script + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX run: | - .github/workflows/scripts/check_diff.sh \ - ${{ github.event.pull_request.head.ref }} \ - ${{ github.event.pull_request.base.ref }} \ - ${{ github.event.pull_request.head.repo.clone_url }} - outputs: - skip: ${{ env.SKIP_CHECKS }} - - build_UB_sanitizer: - name: Clang UB sanitizer - runs-on: ubuntu-22.04 - container: ubuntu:23.10 - needs: skip_checks - if: ${{ github.event.pull_request.draft == false && needs.skip_checks.outputs.skip == false }} - env: - CC: clang - CXX: clang++ - # On CI for this test, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - .github/workflows/dependencies/clang17.sh - - name: CCache Cache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: build WarpX - run: | - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=10 - export CCACHE_MAXSIZE=100M - ccache -z - - export CXX=$(which clang++-17) - export CC=$(which clang-17) - export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all" - - cmake -S . -B build \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE \ - -DWarpX_PARTICLE_PRECISION=SINGLE - cmake --build build -j 4 - - ccache -s - du -hs ~/.cache/ccache - - - name: run with UB sanitizer - run: | - - # We need these two lines because these tests run inside a docker container - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - - export OMP_NUM_THREADS=2 - - #MPI implementations often leak memory - export "ASAN_OPTIONS=detect_leaks=0" - - mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz - mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d - mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d - mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d - - build_thread_sanitizer: - name: Clang thread sanitizer - runs-on: ubuntu-22.04 - container: ubuntu:23.10 - needs: skip_checks - if: ${{ github.event.pull_request.draft == false && needs.skip_checks.outputs.skip == false }} - env: - CC: clang - CXX: clang++ - # On CI for this test, Ninja is slower than the default: - #CMAKE_GENERATOR: Ninja - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - .github/workflows/dependencies/clang17.sh - - name: CCache Cache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: build WarpX - run: | - export CCACHE_COMPRESS=1 - export CCACHE_COMPRESSLEVEL=10 - export CCACHE_MAXSIZE=100M - ccache -z - - export CXX=$(which clang++-17) - export CC=$(which clang-17) - export CXXFLAGS="-fsanitize=thread" - - cmake -S . -B build \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_EB=OFF \ - -DWarpX_PRECISION=DOUBLE \ - -DWarpX_PARTICLE_PRECISION=DOUBLE - cmake --build build -j 4 - - cmake -S . -B build_EB \ - -GNinja \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DWarpX_DIMS="2" \ - -DWarpX_FFT=ON \ - -DWarpX_QED=ON \ - -DWarpX_QED_TABLE_GEN=ON \ - -DWarpX_OPENPMD=ON \ - -DWarpX_EB=ON \ - -DWarpX_PRECISION=DOUBLE \ - -DWarpX_PARTICLE_PRECISION=DOUBLE - cmake --build build_EB -j 4 - - ccache -s - du -hs ~/.cache/ccache - - - name: run with thread sanitizer - run: | - export PMIX_MCA_gds=hash - export TSAN_OPTIONS='ignore_noninstrumented_modules=1' - export ARCHER_OPTIONS="verbose=1" - - # We need these two lines because these tests run inside a docker container - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - - export OMP_NUM_THREADS=2 - - mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0 - mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d warpx.serialize_initial_conditions = 0 - mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d warpx.serialize_initial_conditions = 0 - mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d warpx.serialize_initial_conditions = 0 - - git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data - cd Examples/Tests/embedded_circle - - ulimit -c unlimited - - mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_test_2d_embedded_circle warpx.serialize_initial_conditions = 0 + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all" + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE \ + -DWarpX_PARTICLE_PRECISION=SINGLE + cmake --build build -j 4 + ccache -s + du -hs ~/.cache/ccache + - name: run with UB sanitizer + run: | + # We need these two lines because these tests run inside a docker container + export OMPI_ALLOW_RUN_AS_ROOT=1 + export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + export OMP_NUM_THREADS=2 + #MPI implementations often leak memory + export "ASAN_OPTIONS=detect_leaks=0" + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d + - build_thread_sanitizer: + name: Clang thread sanitizer + runs-on: ubuntu-22.04 + container: ubuntu:23.10 + needs: skip_checks + if: ${{ github.event.pull_request.draft == false && needs.skip_checks.outputs.skip == false }} + env: + CC: clang + CXX: clang++ + # On CI for this test, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + .github/workflows/dependencies/clang17.sh + - name: CCache Cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + export CXX=$(which clang++-17) + export CC=$(which clang-17) + export CXXFLAGS="-fsanitize=thread" + cmake -S . -B build \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="RZ;1;2;3" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=OFF \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build -j 4 + cmake -S . -B build_EB \ + -GNinja \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DWarpX_DIMS="2" \ + -DWarpX_FFT=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_EB=ON \ + -DWarpX_PRECISION=DOUBLE \ + -DWarpX_PARTICLE_PRECISION=DOUBLE + cmake --build build_EB -j 4 + ccache -s + du -hs ~/.cache/ccache + - name: run with thread sanitizer + run: | + export PMIX_MCA_gds=hash + export TSAN_OPTIONS='ignore_noninstrumented_modules=1' + export ARCHER_OPTIONS="verbose=1" + # We need these two lines because these tests run inside a docker container + export OMPI_ALLOW_RUN_AS_ROOT=1 + export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + export OMP_NUM_THREADS=2 + mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d warpx.serialize_initial_conditions = 0 + mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d warpx.serialize_initial_conditions = 0 + git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data + cd Examples/Tests/embedded_circle + ulimit -c unlimited + mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_test_2d_embedded_circle warpx.serialize_initial_conditions = 0 diff --git a/.github/workflows/skip_checks.yml b/.github/workflows/skip_checks.yml new file mode 100644 index 00000000000..4ec12a4ddb0 --- /dev/null +++ b/.github/workflows/skip_checks.yml @@ -0,0 +1,17 @@ +name: Skip checks +jobs: + skip_checks: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Run PR analysis script + run: | + .github/workflows/scripts/check_diff.sh \ + ${{ github.event.pull_request.head.ref }} \ + ${{ github.event.pull_request.base.ref }} \ + ${{ github.event.pull_request.head.repo.clone_url }} + outputs: + skip: ${{ env.SKIP_CHECKS }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9a0a8bfdea..1383623825c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: mixed-line-ending - id: check-json - id: check-toml - - id: check-yaml + #- id: check-yaml - id: check-added-large-files args: ['--maxkb=40'] - id: requirements-txt-fixer