Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MINIFICPP-2522 Various CI improvements #1936

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

MINIFICPP-2522 Various CI improvements #1936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
616375b
MINIFICPP-2522 Various CI improvements
martinzink Jan 29, 2025
58dc95b
remove ubuntu 24.04
martinzink Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 16 additions & 114 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ env:
CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
jobs:
macos_xcode:
name: "macos-xcode"
runs-on: macos-14
name: "macOS 15 aarch64"
runs-on: macos-15
timeout-minutes: 180
env:
MACOS_MINIFI_OPTIONS: >-
Expand Down Expand Up @@ -92,7 +92,7 @@ jobs:
run: |
# Set core file size limit to unlimited
ulimit -c unlimited
ctest -j4 --output-on-failure --timeout 300
ctest --timeout 300 -j4 --output-on-failure --timeout 300
working-directory: build
- name: linter
run: ninja linter
Expand All @@ -112,8 +112,8 @@ jobs:
name: macos-binaries
path: build/bin
windows_VS2022:
name: "windows-2022"
runs-on: windows-2022
name: "Windows Server 2025 x86_64"
runs-on: windows-2025
timeout-minutes: 240
env:
WINDOWS_MINIFI_OPTIONS: >-
Expand Down Expand Up @@ -228,107 +228,9 @@ jobs:
with:
name: cpack_error
path: D:/a/nifi-minifi-cpp/nifi-minifi-cpp/build/_CPack_Packages/win64/WIX/wix.log
ubuntu_24_04:
name: "ubuntu-24.04"
runs-on: ubuntu-24.04
timeout-minutes: 120
env:
UBUNTU_GCC_MINIFI_OPTIONS: >-
-DCMAKE_BUILD_TYPE=Release
-DCI_BUILD=OFF
-DCUSTOM_MALLOC=OFF
-DDOCKER_BUILD_ONLY=OFF
-DDOCKER_PUSH=OFF
-DDOCKER_SKIP_TESTS=ON
-DENABLE_ALL=OFF
-DENABLE_AWS=OFF
-DENABLE_AZURE=OFF
-DENABLE_BUSTACHE=ON
-DENABLE_BZIP2=ON
-DENABLE_CIVET=ON
-DENABLE_CONTROLLER=ON
-DENABLE_COUCHBASE=OFF
-DENABLE_ELASTICSEARCH=OFF
-DENABLE_ENCRYPT_CONFIG=ON
-DENABLE_EXPRESSION_LANGUAGE=ON
-DENABLE_GCP=OFF
-DENABLE_KUBERNETES=OFF
-DENABLE_LIBARCHIVE=ON
-DENABLE_KAFKA=OFF
-DENABLE_LUA_SCRIPTING=OFF
-DENABLE_LZMA=ON
-DENABLE_MQTT=OFF
-DENABLE_OPC=OFF
-DENABLE_OPENCV=OFF
-DENABLE_OPS=ON
-DENABLE_PROCFS=OFF
-DENABLE_PROMETHEUS=ON
-DENABLE_PYTHON_SCRIPTING=OFF
-DENABLE_ROCKSDB=ON
-DENABLE_SFTP=ON
-DENABLE_SPLUNK=OFF
-DENABLE_SQL=OFF
-DENABLE_SYSTEMD=ON
-DENABLE_TEST_PROCESSORS=OFF
-DFORCE_COLORED_OUTPUT=ON
-DMINIFI_FAIL_ON_WARNINGS=ON
-DPORTABLE=ON
-DSKIP_TESTS=OFF
-DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF
-DUSE_SHARED_LIBS=ON
steps:
- id: checkout
uses: actions/checkout@v4
- name: cache restore
uses: actions/cache/restore@v4
with:
path: ${{ env.CCACHE_DIR }}
key: ubuntu-24.04-ccache-${{github.ref}}-${{github.sha}}
restore-keys: |
ubuntu-24.04-ccache-${{github.ref}}-
ubuntu-24.04-ccache-refs/heads/main-
- id: install_deps
run: |
sudo apt update
sudo apt install -y ccache libfl-dev python3 python3-venv
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
- name: build
run: |
python3 -m venv venv && source venv/bin/activate \
&& pip install -r requirements.txt \
&& python main.py --noninteractive --minifi-options="${UBUNTU_GCC_MINIFI_OPTIONS}" --cmake-options="-DSTRICT_GSL_CHECKS=AUDIT"
working-directory: bootstrap
- name: cache save
uses: actions/cache/save@v4
if: always()
with:
path: ${{ env.CCACHE_DIR }}
key: ubuntu-24.04-ccache-${{github.ref}}-${{github.sha}}
- name: test
id: test
run: |
# Set core file size limit to unlimited
ulimit -c unlimited
ctest -j$(nproc) --output-on-failure
working-directory: build
- name: check-cores
if: ${{ failure() && steps.test.conclusion == 'failure' }}
run: |
if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
- uses: actions/upload-artifact@v4
if: ${{ failure() && env.CORES_EXIST == 'true' }}
with:
name: ubuntu-coredumps
path: /var/lib/apport/coredump/
- uses: actions/upload-artifact@v4
if: ${{ failure() && env.CORES_EXIST == 'true' }}
with:
name: ubuntu-binaries
path: build/bin
ubuntu_22_04_clang:
name: "ubuntu-22.04-clang"
runs-on: ubuntu-22.04
ubuntu_22_04_clang_arm:
name: "Ubuntu 22.04 clang aarch64"
runs-on: ubuntu-22.04-arm
timeout-minutes: 240
env:
UBUNTU_CLANG_MINIFI_OPTIONS: >-
Expand Down Expand Up @@ -383,10 +285,10 @@ jobs:
uses: actions/cache/restore@v4
with:
path: ${{ env.CCACHE_DIR }}
key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}}
restore-keys: |
ubuntu-22.04-all-clang-ccache-${{github.ref}}-
ubuntu-22.04-all-clang-ccache-refs/heads/main-
ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-
ubuntu-22.04-arm-all-clang-ccache-refs/heads/main-
- id: install_deps
run: |
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
Expand Down Expand Up @@ -423,13 +325,13 @@ jobs:
if: always()
with:
path: ${{ env.CCACHE_DIR }}
key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}}
- name: test
id: test
run: |
# Set core file size limit to unlimited
ulimit -c unlimited
ctest -j$(nproc) --output-on-failure
ctest --timeout 300 -j$(nproc) --output-on-failure
working-directory: build
- name: linter
run: cmake --build . --target linter
Expand Down Expand Up @@ -474,7 +376,7 @@ jobs:
name: ubuntu-clang-binaries
path: build/bin
rocky:
name: "rocky"
name: "Rocky (docker) x86_64"
runs-on: ubuntu-24.04
timeout-minutes: 180
steps:
Expand Down Expand Up @@ -543,7 +445,7 @@ jobs:
name: rocky-binaries
path: /tmp/bin
docker_build:
name: "Docker build for integration tests"
name: "Docker build for integration tests (x86_64)"
runs-on: ubuntu-22.04
timeout-minutes: 180
steps:
Expand Down Expand Up @@ -587,7 +489,7 @@ jobs:
name: minifi_docker
path: build/minifi_docker.tar
docker_tests:
name: "Docker integration tests"
name: "Docker integration tests (x86_64)"
needs: docker_build
runs-on: ubuntu-24.04
timeout-minutes: 180
Expand Down
46 changes: 8 additions & 38 deletions .github/workflows/verify-package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
name: "MiNiFi-CPP Verify Package"
on: [workflow_dispatch]
on:
schedule:
- cron: '0 0 * * 0'
workflow_dispatch:
env:
DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
-DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
-DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON
CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This job is not frequent enough to warrant caching

jobs:
rocky-x86-64-build:
name: "Build package (Rocky 8 x86_64)"
Expand All @@ -13,29 +15,13 @@ jobs:
steps:
- id: checkout
uses: actions/checkout@v4
- name: cache restore
uses: actions/cache/restore@v4
with:
path: ~/.ccache
key: verify-package-ccache-${{github.ref}}-${{github.sha}}
restore-keys: |
verify-package-ccache-${{github.ref}}-
verify-package-ccache-refs/heads/main-
- id: install_deps
run: |
sudo apt update
sudo apt install -y ccache cmake
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
sudo apt install -y cmake
- id: build
run: |
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky
- name: cache save
uses: actions/cache/save@v4
if: always()
with:
path: ${{ env.CCACHE_DIR }}
key: verify-package-ccache-${{github.ref}}-${{github.sha}}
mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky
- uses: actions/upload-artifact@v4
with:
name: minifi-x86-64-tar
Expand All @@ -49,29 +35,13 @@ jobs:
steps:
- id: checkout
uses: actions/checkout@v4
- name: cache restore
uses: actions/cache/restore@v4
with:
path: ~/.ccache
key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
restore-keys: |
verify-package-arm-ccache-${{github.ref}}-
verify-package-arm-ccache-refs/heads/main-
- id: install_deps
run: |
sudo apt update
sudo apt install -y ccache cmake
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
sudo apt install -y cmake
- id: build
run: |
if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky
- name: cache save
uses: actions/cache/save@v4
if: always()
with:
path: ${{ env.CCACHE_DIR }}
key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky
- uses: actions/upload-artifact@v4
with:
name: minifi-aarch64-tar
Expand Down
3 changes: 1 addition & 2 deletions cmake/DockerConfig.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
endif()
endforeach()

set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
set(DISABLED_TAGS "SKIP_CI")

add_custom_target(
docker-verify
Expand All @@ -172,5 +172,4 @@ function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_P
${CMAKE_BINARY_DIR})
endfunction()

include(VerifyPythonCompatibility)
include(VerifyPackageWithDocker)
2 changes: 1 addition & 1 deletion cmake/VerifyPackageWithDocker.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX)
endif()
endforeach()

set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
set(DISABLED_TAGS "SKIP_CI")

add_custom_target(
docker-verify-${TAG_PREFIX}
Expand Down
72 changes: 0 additions & 72 deletions cmake/VerifyPythonCompatibility.cmake
View file
Open in desktop

This file was deleted.

Loading