From e8f9cbebc027a979249a877f62c796a54d7e54bc Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 29 Jan 2025 13:11:19 +0100 Subject: [PATCH] MINIFICPP-2522 Various CI improvements --- .github/workflows/ci.yml | 14 ++-- .github/workflows/verify-package.yml | 46 +++--------- cmake/DockerConfig.cmake | 3 +- cmake/VerifyPackageWithDocker.cmake | 2 +- cmake/VerifyPythonCompatibility.cmake | 72 ------------------- .../integration/features/couchbase.feature | 8 +++ .../features/python_with_modules.feature | 30 -------- 7 files changed, 25 insertions(+), 150 deletions(-) delete mode 100644 cmake/VerifyPythonCompatibility.cmake delete mode 100644 docker/test/integration/features/python_with_modules.feature diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6b3d97c37..988008333b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,9 +326,9 @@ jobs: with: name: ubuntu-binaries path: build/bin - ubuntu_22_04_clang: - name: "ubuntu-22.04-clang" - runs-on: ubuntu-22.04 + ubuntu_22_04_clang_arm: + name: "ubuntu-22.04-clang-arm" + runs-on: ubuntu-22.04-arm timeout-minutes: 240 env: UBUNTU_CLANG_MINIFI_OPTIONS: >- @@ -383,10 +383,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ env.CCACHE_DIR }} - key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}} + key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}} restore-keys: | - ubuntu-22.04-all-clang-ccache-${{github.ref}}- - ubuntu-22.04-all-clang-ccache-refs/heads/main- + ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}- + ubuntu-22.04-arm-all-clang-ccache-refs/heads/main- - id: install_deps run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - @@ -423,7 +423,7 @@ jobs: if: always() with: path: ${{ env.CCACHE_DIR }} - key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}} + key: ubuntu-22.04-arm-all-clang-ccache-${{github.ref}}-${{github.sha}} - name: test id: test run: | diff --git a/.github/workflows/verify-package.yml b/.github/workflows/verify-package.yml index b5a788324b..99df50230e 100644 --- a/.github/workflows/verify-package.yml +++ b/.github/workflows/verify-package.yml @@ -1,10 +1,12 @@ name: "MiNiFi-CPP Verify Package" -on: [workflow_dispatch] +on: + schedule: + - cron: '0 0 * * 0' + workflow_dispatch: env: DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ -DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON - CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache jobs: rocky-x86-64-build: name: "Build package (Rocky 8 x86_64)" @@ -13,29 +15,13 @@ jobs: steps: - id: checkout uses: actions/checkout@v4 - - name: cache restore - uses: actions/cache/restore@v4 - with: - path: ~/.ccache - key: verify-package-ccache-${{github.ref}}-${{github.sha}} - restore-keys: | - verify-package-ccache-${{github.ref}}- - verify-package-ccache-refs/heads/main- - id: install_deps run: | sudo apt update - sudo apt install -y ccache cmake - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + sudo apt install -y cmake - id: build run: | - if [ -d ~/.ccache ]; then mv ~/.ccache .; fi - mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky - - name: cache save - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ env.CCACHE_DIR }} - key: verify-package-ccache-${{github.ref}}-${{github.sha}} + mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky - uses: actions/upload-artifact@v4 with: name: minifi-x86-64-tar @@ -49,29 +35,13 @@ jobs: steps: - id: checkout uses: actions/checkout@v4 - - name: cache restore - uses: actions/cache/restore@v4 - with: - path: ~/.ccache - key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}} - restore-keys: | - verify-package-arm-ccache-${{github.ref}}- - verify-package-arm-ccache-refs/heads/main- - id: install_deps run: | sudo apt update - sudo apt install -y ccache cmake - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + sudo apt install -y cmake - id: build run: | - if [ -d ~/.ccache ]; then mv ~/.ccache .; fi - mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky - - name: cache save - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ env.CCACHE_DIR }} - key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}} + mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} .. && make rocky - uses: actions/upload-artifact@v4 with: name: minifi-aarch64-tar diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 956d0c097d..d1d88d65e9 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -152,7 +152,7 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) endif() endforeach() - set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY") + set(DISABLED_TAGS "SKIP_CI") add_custom_target( docker-verify @@ -172,5 +172,4 @@ function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_P ${CMAKE_BINARY_DIR}) endfunction() -include(VerifyPythonCompatibility) include(VerifyPackageWithDocker) diff --git a/cmake/VerifyPackageWithDocker.cmake b/cmake/VerifyPackageWithDocker.cmake index af201791a9..0a76710f8e 100644 --- a/cmake/VerifyPackageWithDocker.cmake +++ b/cmake/VerifyPackageWithDocker.cmake @@ -28,7 +28,7 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX) endif() endforeach() - set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY") + set(DISABLED_TAGS "SKIP_CI") add_custom_target( docker-verify-${TAG_PREFIX} diff --git a/cmake/VerifyPythonCompatibility.cmake b/cmake/VerifyPythonCompatibility.cmake deleted file mode 100644 index a8b11f5b0f..0000000000 --- a/cmake/VerifyPythonCompatibility.cmake +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if (NOT (ENABLE_ALL OR ENABLE_PYTHON_SCRIPTING)) - return() -endif() - -set(MINIFI_VERSION_STR ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}) - -function(ADD_DOCKER_VERIFY_PYTHON TAG_PREFIX HAS_MODULES) - if (HAS_MODULES) - add_custom_target( - docker-verify-${TAG_PREFIX} - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING) - else() - add_custom_target( - docker-verify-${TAG_PREFIX} - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING --tags_to_exclude NEEDS_NUMPY) - endif() -endfunction() - -function(ADD_CONDA_TO_DOCKER TAG_PREFIX) - add_custom_target( - conda_${TAG_PREFIX}_from_rocky_package - COMMAND DOCKER_BUILDKIT=1 docker build - --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} - -t apacheminificpp:conda_${TAG_PREFIX}-${MINIFI_VERSION_STR} - -f ${CMAKE_SOURCE_DIR}/docker/python-verify/conda.Dockerfile - ${CMAKE_BINARY_DIR}) -endfunction() - -function(ADD_VENV_TO_DOCKER TAG_PREFIX) - add_custom_target( - venv_${TAG_PREFIX}_from_rocky_package - COMMAND DOCKER_BUILDKIT=1 docker build - --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} - -t apacheminificpp:venv_${TAG_PREFIX}-${MINIFI_VERSION_STR} - -f ${CMAKE_SOURCE_DIR}/docker/python-verify/venv.Dockerfile - ${CMAKE_BINARY_DIR}) -endfunction() - - -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye patched_bullseye_py "apt update \\&\\& apt install -y patchelf libpython3-dev python3-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.9.so") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy patched_jammy_py "apt update \\&\\& apt install -y patchelf libpython3.10-dev python3.10-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.10.so.1.0") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8_py "yum install -y python3-libs python3-pip python3-devel gcc-c++ wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9_py "yum install -y python3-libs python3-pip wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy_py "apt update \\&\\& apt install -y wget") -ADD_CONDA_TO_DOCKER(jammy_py) -ADD_VENV_TO_DOCKER(rocky9_py) - -if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) - ADD_DOCKER_VERIFY_PYTHON(rocky8_py FALSE) - ADD_DOCKER_VERIFY_PYTHON(rocky9_py FALSE) - ADD_DOCKER_VERIFY_PYTHON(patched_jammy_py FALSE) - ADD_DOCKER_VERIFY_PYTHON(patched_bullseye_py FALSE) - ADD_DOCKER_VERIFY_PYTHON(conda_jammy_py TRUE) - ADD_DOCKER_VERIFY_PYTHON(venv_rocky9_py TRUE) -endif() diff --git a/docker/test/integration/features/couchbase.feature b/docker/test/integration/features/couchbase.feature index 7b74a6c3dd..90cb0df0de 100644 --- a/docker/test/integration/features/couchbase.feature +++ b/docker/test/integration/features/couchbase.feature @@ -29,6 +29,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the LogAttribute When a Couchbase server is started @@ -53,6 +54,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the LogAttribute When a Couchbase server is started @@ -80,6 +82,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute @@ -109,6 +112,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute @@ -139,6 +143,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute @@ -167,6 +172,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey When a Couchbase server is started @@ -190,6 +196,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is set up up with SSL connection with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute @@ -218,6 +225,7 @@ Feature: Executing Couchbase operations from MiNiFi-C++ And a CouchbaseClusterService is setup up using mTLS authentication with the name "CouchbaseClusterService" And the "success" relationship of the GetFile processor is connected to the PutCouchbaseKey + And the "failure" relationship of the PutCouchbaseKey processor is connected to the PutCouchbaseKey And the "success" relationship of the PutCouchbaseKey processor is connected to the GetCouchbaseKey And the "success" relationship of the GetCouchbaseKey processor is connected to the PutFile And the "success" relationship of the PutFile processor is connected to the LogAttribute diff --git a/docker/test/integration/features/python_with_modules.feature b/docker/test/integration/features/python_with_modules.feature deleted file mode 100644 index 85272a7aa9..0000000000 --- a/docker/test/integration/features/python_with_modules.feature +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -@ENABLE_PYTHON_SCRIPTING -@NEEDS_NUMPY -Feature: MiNiFi can use python modules - Background: - Given the content of "/tmp/output" is monitored - - Scenario: MiNiFi can use python modules - Given the example MiNiFi python processors are present - And a GaussianDistributionWithNumpy processor - And the scheduling period of the GaussianDistributionWithNumpy processor is set to "10 min" - And a PutFile processor with the "Directory" property set to "/tmp/output" - And the "success" relationship of the GaussianDistributionWithNumpy processor is connected to the PutFile - - When the MiNiFi instance starts up - Then 1 flowfile is placed in the monitored directory in 20 seconds