From 406a79a149954ed68215f81e55e174c5f5bf5551 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 15 Oct 2024 14:34:17 +0200 Subject: [PATCH] memcheck --- .github/workflows/memcheck_ci.yml | 88 +++++++++++++++++++ CMakeLists.txt | 4 + extensions/aws/tests/CMakeLists.txt | 1 + extensions/azure/tests/CMakeLists.txt | 1 + extensions/bustache/tests/CMakeLists.txt | 1 + extensions/civetweb/tests/CMakeLists.txt | 1 + extensions/elasticsearch/tests/CMakeLists.txt | 1 + .../execute-process/test/CMakeLists.txt | 2 + .../expression-language/tests/CMakeLists.txt | 1 + extensions/grafana-loki/tests/CMakeLists.txt | 1 + extensions/kubernetes/tests/CMakeLists.txt | 1 + extensions/librdkafka/tests/CMakeLists.txt | 1 + extensions/lua/tests/CMakeLists.txt | 1 + extensions/mqtt/tests/CMakeLists.txt | 1 + extensions/opencv/tests/CMakeLists.txt | 1 + extensions/pdh/tests/CMakeLists.txt | 1 + extensions/procfs/tests/CMakeLists.txt | 1 + extensions/prometheus/tests/CMakeLists.txt | 1 + extensions/script/ExecuteScript.h | 4 +- 19 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/memcheck_ci.yml diff --git a/.github/workflows/memcheck_ci.yml b/.github/workflows/memcheck_ci.yml new file mode 100644 index 0000000000..62268d23e6 --- /dev/null +++ b/.github/workflows/memcheck_ci.yml @@ -0,0 +1,88 @@ +name: "MiNiFi-CPP memcheck" +on: [push, pull_request, workflow_dispatch] +env: + CMAKE_FLAGS: >- + -DCMAKE_BUILD_TYPE=Debug + -DCI_BUILD=OFF + -DCUSTOM_MALLOC=OFF + -DDOCKER_BUILD_ONLY=OFF + -DDOCKER_PUSH=OFF + -DDOCKER_SKIP_TESTS=ON + -DENABLE_ALL=OFF + -DENABLE_AWS=OFF + -DENABLE_AZURE=OFF + -DENABLE_BUSTACHE=ON + -DENABLE_BZIP2=ON + -DENABLE_CIVET=ON + -DENABLE_CONTROLLER=ON + -DENABLE_ELASTICSEARCH=OFF + -DENABLE_ENCRYPT_CONFIG=ON + -DENABLE_EXPRESSION_LANGUAGE=ON + -DENABLE_GCP=OFF + -DENABLE_KUBERNETES=OFF + -DENABLE_LIBARCHIVE=ON + -DENABLE_LIBRDKAFKA=OFF + -DENABLE_LUA_SCRIPTING=OFF + -DENABLE_LZMA=ON + -DENABLE_MQTT=OFF + -DENABLE_OPC=OFF + -DENABLE_OPENCV=OFF + -DENABLE_OPS=ON + -DENABLE_PROCFS=OFF + -DENABLE_PROMETHEUS=ON + -DENABLE_PYTHON_SCRIPTING=OFF + -DENABLE_ROCKSDB=ON + -DENABLE_SFTP=ON + -DENABLE_SPLUNK=OFF + -DENABLE_SQL=OFF + -DENABLE_SYSTEMD=ON + -DENABLE_TEST_PROCESSORS=OFF + -DFORCE_COLORED_OUTPUT=ON + -DMINIFI_FAIL_ON_WARNINGS=ON + -DPORTABLE=ON + -DSKIP_TESTS=OFF + -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF + -DUSE_SHARED_LIBS=ON + SCCACHE_GHA_ENABLE: true + CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache +jobs: + ubuntu_24_04: + name: "ubuntu-24.04" + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - id: checkout + uses: actions/checkout@v4 + - name: cache restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: memcheck-ccache-${{github.ref}}-${{github.sha}} + restore-keys: | + memcheck-ccache-${{github.ref}}- + memcheck-ccache-refs/heads/main- + - id: install_deps + run: | + sudo apt update + sudo apt install -y ccache libfl-dev python3 python3-venv + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null + - name: build + run: | + python3 -m venv venv && source venv/bin/activate \ + && pip install -r requirements.txt \ + && python main.py --noninteractive --minifi-options="${CMAKE_FLAGS}" + working-directory: bootstrap + - name: cache save + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ env.CCACHE_DIR }} + key: memcheck-ccache-${{github.ref}}-${{github.sha}} + - name: test + id: test + run: | + # Set core file size limit to unlimited + ulimit -c unlimited + ctest -j$(nproc) -L memchecked -T memcheck + working-directory: build \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 1699e95f99..e852e6922f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -667,6 +667,10 @@ include(CPack) ### include modules if (NOT SKIP_TESTS) + enable_testing() + set(BUILD_TESTING ON) + set(CTEST_NEW_FORMAT true) + include(CTest) include(BuildTests) add_subdirectory("${TEST_DIR}/libtest") diff --git a/extensions/aws/tests/CMakeLists.txt b/extensions/aws/tests/CMakeLists.txt index d26c0ebee9..542211aeee 100644 --- a/extensions/aws/tests/CMakeLists.txt +++ b/extensions/aws/tests/CMakeLists.txt @@ -35,5 +35,6 @@ FOREACH(testfile ${AWS_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-expression-language-extensions) MATH(EXPR AWS_TEST_COUNT "${AWS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "aws") ENDFOREACH() message("-- Finished building ${AWS_TEST_COUNT} AWS related test file(s)...") diff --git a/extensions/azure/tests/CMakeLists.txt b/extensions/azure/tests/CMakeLists.txt index 730cf34074..55ec534a2f 100644 --- a/extensions/azure/tests/CMakeLists.txt +++ b/extensions/azure/tests/CMakeLists.txt @@ -35,5 +35,6 @@ FOREACH(testfile ${AZURE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-expression-language-extensions) MATH(EXPR AZURE_TEST_COUNT "${AZURE_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "azure;memchecked") ENDFOREACH() message("-- Finished building ${AZURE_TEST_COUNT} Azure related test file(s)...") diff --git a/extensions/bustache/tests/CMakeLists.txt b/extensions/bustache/tests/CMakeLists.txt index 4f493df658..eb689ec2a9 100644 --- a/extensions/bustache/tests/CMakeLists.txt +++ b/extensions/bustache/tests/CMakeLists.txt @@ -33,5 +33,6 @@ FOREACH(testfile ${BUSTACHE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR BUSTACHE-EXTENSIONS_TEST_COUNT "${BUSTACHE-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "bustache;memchecked") ENDFOREACH() message("-- Finished building ${BUSTACHE-EXTENSIONS_TEST_COUNT} Bustache related test file(s)...") diff --git a/extensions/civetweb/tests/CMakeLists.txt b/extensions/civetweb/tests/CMakeLists.txt index 95f7af7d87..6d9ad431a2 100644 --- a/extensions/civetweb/tests/CMakeLists.txt +++ b/extensions/civetweb/tests/CMakeLists.txt @@ -40,5 +40,6 @@ FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS}) "${CMAKE_SOURCE_DIR}/extensions/civetweb/tests/resources" "$/resources" ) + set_tests_properties("${testfilename}" PROPERTIES LABELS "civetweb") ENDFOREACH() message("-- Finished building ${CIVETWEB-EXTENSIONS_TEST_COUNT} civetweb related test file(s)...") diff --git a/extensions/elasticsearch/tests/CMakeLists.txt b/extensions/elasticsearch/tests/CMakeLists.txt index 58a2120131..6fbd8b07b0 100644 --- a/extensions/elasticsearch/tests/CMakeLists.txt +++ b/extensions/elasticsearch/tests/CMakeLists.txt @@ -34,5 +34,6 @@ FOREACH(testfile ${ELASTICSEARCH_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR ELASTICSEARCH_TEST_COUNT "${ELASTICSEARCH_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "elasticsearch;memchecked") ENDFOREACH() message("-- Finished building ${ELASTICSEARCH_TEST_COUNT} Elasticsearch related test file(s)...") diff --git a/extensions/execute-process/test/CMakeLists.txt b/extensions/execute-process/test/CMakeLists.txt index 05d5f93480..570410d077 100644 --- a/extensions/execute-process/test/CMakeLists.txt +++ b/extensions/execute-process/test/CMakeLists.txt @@ -25,6 +25,8 @@ add_minifi_executable(ExecuteProcessTests ExecuteProcessTests.cpp) target_link_libraries(ExecuteProcessTests minifi-execute-process minifi-standard-processors Catch2WithMain) target_include_directories(ExecuteProcessTests PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test") add_test(NAME ExecuteProcessTests COMMAND ExecuteProcessTests WORKING_DIRECTORY ${TEST_DIR}) +set_tests_properties(ExecuteProcessTests PROPERTIES LABELS "execute-process;memchecked") + createTests(ExecuteProcessTests) add_dependencies(ExecuteProcessTests EchoParameters) diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt index 2f96ee3df2..d01c18b65e 100644 --- a/extensions/expression-language/tests/CMakeLists.txt +++ b/extensions/expression-language/tests/CMakeLists.txt @@ -36,6 +36,7 @@ FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "expression-language;memchecked") ENDFOREACH() diff --git a/extensions/grafana-loki/tests/CMakeLists.txt b/extensions/grafana-loki/tests/CMakeLists.txt index 8aa24fd887..8c17383585 100644 --- a/extensions/grafana-loki/tests/CMakeLists.txt +++ b/extensions/grafana-loki/tests/CMakeLists.txt @@ -40,5 +40,6 @@ FOREACH(testfile ${GRAFANA_LOKI_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR GRAFANA_LOKI_TEST_COUNT "${GRAFANA_LOKI_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "grafana-loki;memchecked") ENDFOREACH() message("-- Finished building ${GRAFANA_LOKI_TEST_COUNT} Grafana Loki related test file(s)...") diff --git a/extensions/kubernetes/tests/CMakeLists.txt b/extensions/kubernetes/tests/CMakeLists.txt index d5dd9abcab..d8b80e3426 100644 --- a/extensions/kubernetes/tests/CMakeLists.txt +++ b/extensions/kubernetes/tests/CMakeLists.txt @@ -29,5 +29,6 @@ FOREACH(TEST_FILE ${KUBERNETES_UNIT_TESTS}) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR KUBERNETES_UNIT_TEST_COUNT "${KUBERNETES_UNIT_TEST_COUNT}+1") + set_tests_properties("${TEST_TARGET}" PROPERTIES LABELS "kubernetes;memchecked") ENDFOREACH() message("-- Finished building ${KUBERNETES_UNIT_TEST_COUNT} Kubernetes related test file(s)...") diff --git a/extensions/librdkafka/tests/CMakeLists.txt b/extensions/librdkafka/tests/CMakeLists.txt index 7f8053884c..8bf4579f1d 100644 --- a/extensions/librdkafka/tests/CMakeLists.txt +++ b/extensions/librdkafka/tests/CMakeLists.txt @@ -30,5 +30,6 @@ FOREACH(testfile ${KAFKA_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "librdkafka;memchecked") ENDFOREACH() message("-- Finished building ${KAFKA-EXTENSIONS_TEST_COUNT} Lib Kafka related test file(s)...") diff --git a/extensions/lua/tests/CMakeLists.txt b/extensions/lua/tests/CMakeLists.txt index 38d6979906..abe00a1cbb 100644 --- a/extensions/lua/tests/CMakeLists.txt +++ b/extensions/lua/tests/CMakeLists.txt @@ -31,6 +31,7 @@ FOREACH(testfile ${EXECUTESCRIPT_LUA_TESTS}) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "lua;memchecked") ENDFOREACH() copyTestResources(${CMAKE_CURRENT_SOURCE_DIR}/test_lua_scripts/ ${CMAKE_BINARY_DIR}/bin/resources/test_lua_scripts/) diff --git a/extensions/mqtt/tests/CMakeLists.txt b/extensions/mqtt/tests/CMakeLists.txt index d5b408a64a..0969fe635a 100644 --- a/extensions/mqtt/tests/CMakeLists.txt +++ b/extensions/mqtt/tests/CMakeLists.txt @@ -29,6 +29,7 @@ FOREACH(testfile ${MQTT_TESTS}) target_link_libraries(${testfilename} minifi-mqtt-extensions) target_link_libraries(${testfilename} minifi-standard-processors) add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "mqtt;memchecked") ENDFOREACH() list(LENGTH MQTT_TESTS TEST_COUNT) diff --git a/extensions/opencv/tests/CMakeLists.txt b/extensions/opencv/tests/CMakeLists.txt index d095c7e9a2..870f29c0e3 100644 --- a/extensions/opencv/tests/CMakeLists.txt +++ b/extensions/opencv/tests/CMakeLists.txt @@ -32,5 +32,6 @@ FOREACH(testfile ${OPENCV_TESTS}) MATH(EXPR OPENCV_TEST_COUNT "${OPENCV_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) target_link_libraries(${testfilename} Catch2WithMain) + set_tests_properties("${testfilename}" PROPERTIES LABELS "opencv;memchecked") ENDFOREACH() message("-- Finished building ${OPENCV_TEST_COUNT} OpenCV related test file(s)...") diff --git a/extensions/pdh/tests/CMakeLists.txt b/extensions/pdh/tests/CMakeLists.txt index 63686553fa..2ff8ddee51 100644 --- a/extensions/pdh/tests/CMakeLists.txt +++ b/extensions/pdh/tests/CMakeLists.txt @@ -31,5 +31,6 @@ FOREACH(testfile ${PDH_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PDH_TEST_COUNT "${PDH_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + set_tests_properties("${testfilename}" PROPERTIES LABELS "phd;memchecked") ENDFOREACH() message("-- Finished building ${PDH_TEST_COUNT} PDH related test file(s)...") diff --git a/extensions/procfs/tests/CMakeLists.txt b/extensions/procfs/tests/CMakeLists.txt index ad51a869d3..054bce8d64 100644 --- a/extensions/procfs/tests/CMakeLists.txt +++ b/extensions/procfs/tests/CMakeLists.txt @@ -32,6 +32,7 @@ FOREACH(testfile ${PROCFS_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PROCFS_TEST_COUNT "${PROCFS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") + set_tests_properties("${testfilename}" PROPERTIES LABELS "procfs;memchecked") ENDFOREACH() file(COPY "${CMAKE_SOURCE_DIR}/extensions/procfs/tests/mockprocfs_t0" DESTINATION "${CMAKE_BINARY_DIR}/bin/procfs-test/") file(COPY "${CMAKE_SOURCE_DIR}/extensions/procfs/tests/mockprocfs_t1" DESTINATION "${CMAKE_BINARY_DIR}/bin/procfs-test/") diff --git a/extensions/prometheus/tests/CMakeLists.txt b/extensions/prometheus/tests/CMakeLists.txt index 29b3185c43..72420828f2 100644 --- a/extensions/prometheus/tests/CMakeLists.txt +++ b/extensions/prometheus/tests/CMakeLists.txt @@ -29,5 +29,6 @@ FOREACH(testfile ${PROMETHEUS_TESTS}) target_link_libraries(${testfilename} minifi-prometheus) MATH(EXPR PROMETHEUS_TEST_COUNT "${PROMETHEUS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") + set_tests_properties("${testfilename}" PROPERTIES LABELS "prometheus;memchecked") ENDFOREACH() message("-- Finished building ${PROMETHEUS_TEST_COUNT} Prometheus related test file(s)...") diff --git a/extensions/script/ExecuteScript.h b/extensions/script/ExecuteScript.h index 6a38651ee8..1838b48385 100644 --- a/extensions/script/ExecuteScript.h +++ b/extensions/script/ExecuteScript.h @@ -94,7 +94,9 @@ class ExecuteScript : public core::Processor { void initialize() override; void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; void onTrigger(core::ProcessContext& context, core::ProcessSession& session) override; - + void notifyStop() override { + script_executor_.reset(); + } private: std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_);