MINIFICPP-2522 Various CI improvements

martinzink · Feb 19, 2025 · e8f9cbe · e8f9cbe
1 parent 7b6b5de
commit e8f9cbe
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 150 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -326,9 +326,9 @@ jobs:
         with:
           name: ubuntu-binaries
           path: build/bin
-  ubuntu_22_04_clang:
-    name: "ubuntu-22.04-clang"
-    runs-on: ubuntu-22.04
+  ubuntu_22_04_clang_arm:
+    name: "ubuntu-22.04-clang-arm"
+    runs-on: ubuntu-22.04-arm
     timeout-minutes: 240
     env:
       UBUNTU_CLANG_MINIFI_OPTIONS: >-
@@ -383,10 +383,10 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
+          key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}}
           restore-keys: |
-            ubuntu-22.04-all-clang-ccache-${{github.ref}}-
-            ubuntu-22.04-all-clang-ccache-refs/heads/main-
+            ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-
+            ubuntu-22.04-arm-all-clang-ccache-refs/heads/main-
       - id: install_deps
         run: |
           wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
@@ -423,7 +423,7 @@ jobs:
         if: always()
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
+          key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}}
       - name: test
         id: test
         run: |

diff --git a/.github/workflows/verify-package.yml b/.github/workflows/verify-package.yml
@@ -1,10 +1,12 @@
 name: "MiNiFi-CPP Verify Package"
-on: [workflow_dispatch]
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
 env:
   DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
     -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
     -DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON
-  CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
 jobs:
   rocky-x86-64-build:
     name: "Build package (Rocky 8 x86_64)"
@@ -13,29 +15,13 @@ jobs:
     steps:
       - id: checkout
         uses: actions/checkout@v4
-      - name: cache restore
-        uses: actions/cache/restore@v4
-        with:
-          path: ~/.ccache
-          key: verify-package-ccache-${{github.ref}}-${{github.sha}}
-          restore-keys: |
-            verify-package-ccache-${{github.ref}}-
-            verify-package-ccache-refs/heads/main-
       - id: install_deps
         run: |
           sudo apt update
-          sudo apt install -y ccache cmake
-          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+          sudo apt install -y cmake
       - id: build
         run: |
-          if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
-          mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky
-      - name: cache save
-        uses: actions/cache/save@v4
-        if: always()
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: verify-package-ccache-${{github.ref}}-${{github.sha}}
+          mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky
       - uses: actions/upload-artifact@v4
         with:
           name: minifi-x86-64-tar
@@ -49,29 +35,13 @@ jobs:
     steps:
       - id: checkout
         uses: actions/checkout@v4
-      - name: cache restore
-        uses: actions/cache/restore@v4
-        with:
-          path: ~/.ccache
-          key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
-          restore-keys: |
-            verify-package-arm-ccache-${{github.ref}}-
-            verify-package-arm-ccache-refs/heads/main-
       - id: install_deps
         run: |
           sudo apt update
-          sudo apt install -y ccache cmake
-          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+          sudo apt install -y cmake
       - id: build
         run: |
-          if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
-          mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky
-      - name: cache save
-        uses: actions/cache/save@v4
-        if: always()
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
+          mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky
       - uses: actions/upload-artifact@v4
         with:
           name: minifi-aarch64-tar

diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
@@ -152,7 +152,7 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
         endif()
     endforeach()
 
-    set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
+    set(DISABLED_TAGS "SKIP_CI")
 
     add_custom_target(
         docker-verify
@@ -172,5 +172,4 @@ function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_P
             ${CMAKE_BINARY_DIR})
 endfunction()
 
-include(VerifyPythonCompatibility)
 include(VerifyPackageWithDocker)
diff --git a/cmake/VerifyPackageWithDocker.cmake b/cmake/VerifyPackageWithDocker.cmake
@@ -28,7 +28,7 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX)
         endif()
     endforeach()
 
-    set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
+    set(DISABLED_TAGS "SKIP_CI")
 
     add_custom_target(
             docker-verify-${TAG_PREFIX}

diff --git a/cmake/VerifyPythonCompatibility.cmake b/cmake/VerifyPythonCompatibility.cmake
diff --git a/docker/test/integration/features/couchbase.feature b/docker/test/integration/features/couchbase.feature
@@ -29,6 +29,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the LogAttribute
 
     When a Couchbase server is started
@@ -53,6 +54,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the LogAttribute
 
     When a Couchbase server is started
@@ -80,6 +82,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
     And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile
     And the "success" relationship of the PutFile processor is connected to the LogAttribute
@@ -109,6 +112,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
     And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile
     And the "success" relationship of the PutFile processor is connected to the LogAttribute
@@ -139,6 +143,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
     And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile
     And the "success" relationship of the PutFile processor is connected to the LogAttribute
@@ -167,6 +172,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
 
     When a Couchbase server is started
@@ -190,6 +196,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is set up up with SSL connection with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
     And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile
     And the "success" relationship of the PutFile processor is connected to the LogAttribute
@@ -218,6 +225,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++
     And a CouchbaseClusterService is setup up using mTLS authentication with the name "CouchbaseClusterService"
 
     And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey
+    And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey
     And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey
     And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile
     And the "success" relationship of the PutFile processor is connected to the LogAttribute

diff --git a/docker/test/integration/features/python_with_modules.feature b/docker/test/integration/features/python_with_modules.feature