MINIFICPP-2475: Fix CivetWeb build dependencies

CMakeLists.txt

@@ -350,7 +350,7 @@ if (ENABLE_ALL OR ENABLE_AZURE)
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy")
 endif()
 
-if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_CIVET)
+if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_GRAFANA_LOKI OR ENABLE_CIVET)
     include(GetCivetWeb)
     get_civetweb()
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy")

cmake/CivetWeb.cmake

@@ -14,6 +14,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+if(TARGET civetweb::civetweb-cpp)
+    return()
+endif()
 
 include(FetchContent)
 
@@ -43,3 +46,6 @@ target_compile_definitions(civetweb-c-library PRIVATE SOCKET_TIMEOUT_QUANTUM=200
 
 add_library(civetweb::c-library ALIAS civetweb-c-library)
 add_library(civetweb::civetweb-cpp ALIAS civetweb-cpp)
+
+set(CIVETWEB_INCLUDE_DIR "${civetweb_SOURCE_DIR}/include")
+set(CIVETWEB_INCLUDE_DIRS "${CIVETWEB_INCLUDE_DIR}")

extensions/civetweb/CMakeLists.txt

@@ -30,7 +30,7 @@ target_include_directories(minifi-civet-extensions BEFORE PUBLIC
                            ${CMAKE_SOURCE_DIR}/libminifi/include
                            ${CMAKE_SOURCE_DIR}/libminifi/include/core
                            ${CMAKE_SOURCE_DIR}/thirdparty/
-                           ${civetweb_SOURCE_DIR}/include
+                           ${CIVETWEB_INCLUDE_DIRS}/include
                            ./include)
 target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp)
 

extensions/civetweb/tests/CMakeLists.txt

@@ -27,8 +27,9 @@ FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS})
 
     target_link_libraries(${testfilename} minifi-civet-extensions)
     target_link_libraries(${testfilename} minifi-standard-processors)
+    target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}")
 
-    createTests("${testfilename}")
+    createIntegrationTests("${testfilename}")
     target_link_libraries(${testfilename} Catch2WithMain)
     MATH(EXPR CIVETWEB-EXTENSIONS_TEST_COUNT "${CIVETWEB-EXTENSIONS_TEST_COUNT}+1")
     add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
@@ -42,4 +43,5 @@ FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS})
         )
     set_tests_properties("${testfilename}" PROPERTIES LABELS "civetweb")
 ENDFOREACH()
+
 message("-- Finished building ${CIVETWEB-EXTENSIONS_TEST_COUNT} civetweb related test file(s)...")

libminifi/test/integration/HttpPostIntegrationTest.cpp → extensions/civetweb/tests/HttpPostIntegrationTest.cpp

@@ -19,7 +19,6 @@
 #include <string>
 #include <iostream>
 #include "processors/InvokeHTTP.h"
-#include "processors/ListenHTTP.h"
 #include "processors/LogAttribute.h"
 #include "unit/TestBase.h"
 #include "core/logging/Logger.h"
@@ -47,8 +46,6 @@ class HttpTestHarness : public HTTPIntegrationBase {
     LogTestController::getInstance().setDebug<core::ProcessContext>();
     LogTestController::getInstance().setTrace<minifi::processors::InvokeHTTP>();
     LogTestController::getInstance().setDebug<minifi::http::HTTPClient>();
-    LogTestController::getInstance().setDebug<minifi::processors::ListenHTTP>();
-    LogTestController::getInstance().setDebug<minifi::processors::ListenHTTP::Handler>();
     LogTestController::getInstance().setDebug<minifi::processors::LogAttribute>();
     LogTestController::getInstance().setDebug<core::Processor>();
     LogTestController::getInstance().setDebug<minifi::ThreadedSchedulingAgent>();

extensions/elasticsearch/tests/CMakeLists.txt

@@ -25,13 +25,10 @@ FOREACH(testfile ${ELASTICSEARCH_TESTS})
     add_minifi_executable("${testfilename}" "${testfile}")
     target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
     target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/elasticsearch")
-    target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/")
+    target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}")
 
     createTests("${testfilename}")
-    target_link_libraries(${testfilename} Catch2WithMain)
-    target_link_libraries(${testfilename} minifi-elasticsearch)
-    target_link_libraries(${testfilename} minifi-civet-extensions)
-    target_link_libraries(${testfilename} minifi-standard-processors)
+    target_link_libraries(${testfilename} Catch2WithMain civetweb::civetweb-cpp minifi-elasticsearch minifi-standard-processors)
     MATH(EXPR ELASTICSEARCH_TEST_COUNT "${ELASTICSEARCH_TEST_COUNT}+1")
     add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
     set_tests_properties("${testfilename}" PROPERTIES LABELS "elasticsearch;memchecked")

extensions/grafana-loki/tests/CMakeLists.txt

@@ -31,13 +31,10 @@ FOREACH(testfile ${GRAFANA_LOKI_TESTS})
     if (ENABLE_GRPC_FOR_LOKI)
         target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${CMAKE_BINARY_DIR}/grafana-loki-protobuf-generated")
     endif()
-    target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/")
+    target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}")
 
     createTests("${testfilename}")
-    target_link_libraries(${testfilename} Catch2WithMain)
-    target_link_libraries(${testfilename} minifi-grafana-loki)
-    target_link_libraries(${testfilename} minifi-civet-extensions)
-    target_link_libraries(${testfilename} minifi-standard-processors)
+    target_link_libraries(${testfilename} Catch2WithMain civetweb::civetweb-cpp minifi-grafana-loki minifi-standard-processors)
     MATH(EXPR GRAFANA_LOKI_TEST_COUNT "${GRAFANA_LOKI_TEST_COUNT}+1")
     add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
     set_tests_properties("${testfilename}" PROPERTIES LABELS "grafana-loki;memchecked")

extensions/splunk/tests/CMakeLists.txt

@@ -25,13 +25,10 @@ FOREACH(testfile ${SPLUNK_TESTS})
     add_minifi_executable("${testfilename}" "${testfile}")
     target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
     target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/splunk")
-    target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/")
+    target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}")
 
     createTests("${testfilename}")
-    target_link_libraries(${testfilename} Catch2WithMain)
-    target_link_libraries(${testfilename} minifi-splunk)
-    target_link_libraries(${testfilename} minifi-civet-extensions)
-    target_link_libraries(${testfilename} minifi-standard-processors)
+    target_link_libraries(${testfilename} Catch2WithMain civetweb::civetweb-cpp minifi-splunk minifi-standard-processors)
     MATH(EXPR SPLUNK_TEST_COUNT "${SPLUNK_TEST_COUNT}+1")
     add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
 ENDFOREACH()

extensions/standard-processors/tests/CMakeLists.txt

@@ -33,7 +33,7 @@ FOREACH(testfile ${PROCESSOR_UNIT_TESTS})
     target_include_directories(${testfilename} BEFORE PRIVATE "../")
     target_include_directories(${testfilename} BEFORE PRIVATE "../processors")
     target_include_directories(${testfilename} BEFORE PRIVATE ./include)
-    target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/")
+    target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}")
     target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language/")
     createTests("${testfilename}")
     target_link_libraries(${testfilename} Catch2WithMain)
@@ -62,7 +62,7 @@ FOREACH(testfile ${PROCESSOR_INTEGRATION_TESTS})
     target_include_directories(${testfilename} BEFORE PRIVATE "../http/")
     target_include_directories(${testfilename} BEFORE PRIVATE "../protocols/")
     target_include_directories(${testfilename} BEFORE PRIVATE "../sitetosite/")
-    target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/")
+    target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}")
     target_include_directories(${testfilename} BEFORE PRIVATE ./include)
     createIntegrationTests("${testfilename}")
     target_link_libraries(${testfilename})

extensions/standard-processors/tests/integration/InvokeHTTPTests.cpp

@@ -18,12 +18,12 @@
 #include <array>
 #include <memory>
 #include <string>
+#include <unordered_map>
 #include "unit/TestBase.h"
 #include "unit/Catch.h"
 #include "core/Core.h"
 #include "http/HTTPClient.h"
 #include "InvokeHTTP.h"
-#include "processors/ListenHTTP.h"
 #include "core/FlowFile.h"
 #include "unit/ProvenanceTestHelper.h"
 #include "core/Processor.h"
@@ -34,61 +34,73 @@
 #include "unit/SingleProcessorTestController.h"
 #include "integration/ConnectionCountingServer.h"
 #include "unit/TestUtils.h"
+#include "integration/TestServer.h"
+#include "utils/StringUtils.h"
 
 namespace org::apache::nifi::minifi::test {
 
+class TestHandler : public CivetHandler {
+ public:
+  bool handleGet(CivetServer*, struct mg_connection* conn) override {
+    headers_.clear();
+    auto req_info = mg_get_request_info(conn);
+    for (int i = 0; i < req_info->num_headers; ++i) {
+      auto header = &req_info->http_headers[i];
+      headers_[std::string(header->name)] = std::string(header->value);
+    }
+    mg_printf(conn, "HTTP/1.1 200 OK\r\n");
+    return true;
+  }
+
+  [[nodiscard]] const std::unordered_map<std::string, std::string>& getHeaders() const {
+    return headers_;
+  }
+
+ private:
+  std::unordered_map<std::string, std::string> headers_;
+};
+
 class TestHTTPServer {
  public:
-  explicit TestHTTPServer(TestController& test_controller) {
-    LogTestController::getInstance().setDebug<org::apache::nifi::minifi::processors::ListenHTTP>();
-    LogTestController::getInstance().setDebug<org::apache::nifi::minifi::processors::LogAttribute>();
-
-    test_plan_ = test_controller.createPlan();
-
-    listen_http_ = dynamic_cast<processors::ListenHTTP*>(test_plan_->addProcessor("ListenHTTP", PROCESSOR_NAME));
-    log_attribute_ = dynamic_cast<processors::LogAttribute*>(test_plan_->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true));
-    REQUIRE(listen_http_);
-    REQUIRE(log_attribute_);
-    test_plan_->setProperty(listen_http_, org::apache::nifi::minifi::processors::ListenHTTP::BasePath, "testytesttest");
-    test_plan_->setProperty(listen_http_, org::apache::nifi::minifi::processors::ListenHTTP::Port, "8681");
-    test_plan_->setProperty(listen_http_, org::apache::nifi::minifi::processors::ListenHTTP::HeadersAsAttributesRegex, ".*");
-    test_plan_->runProcessor(listen_http_);
-    test_plan_->runProcessor(log_attribute_);
-    thread_ = std::thread{[this] {
-      while (running_) {
-        if (listen_http_->isWorkAvailable()) {
-          test_plan_->runProcessor(listen_http_);
-          test_plan_->runProcessor(log_attribute_);
-        }
-      }
-    }};
+  TestHTTPServer() : server_(std::make_unique<TestServer>("8681", "/testytesttest", &handler_)) {
   }
   TestHTTPServer(const TestHTTPServer&) = delete;
   TestHTTPServer(TestHTTPServer&&) = delete;
   TestHTTPServer& operator=(const TestHTTPServer&) = delete;
   TestHTTPServer& operator=(TestHTTPServer&&) = delete;
+  ~TestHTTPServer() = default;
 
-  static constexpr const char* PROCESSOR_NAME = "my_http_server";
   static constexpr const char* URL = "http://localhost:8681/testytesttest";
 
-  ~TestHTTPServer() {
-    running_ = false;
-    thread_.join();
+  [[nodiscard]] const std::unordered_map<std::string, std::string>& getHeaders() const {
+    return handler_.getHeaders();
+  }
+
+  [[nodiscard]] std::vector<std::string> getHeaderKeys() const {
+    std::vector<std::string> keys;
+    for (const auto& [key, _] : handler_.getHeaders()) {
+      keys.push_back(key);
+    }
+    return keys;
+  }
+
+  [[nodiscard]] bool noInvalidHeaderPresent() const {
+    auto header_keys = getHeaderKeys();
+    return std::none_of(header_keys.begin(), header_keys.end(), [] (const auto& key) {
+      return minifi::utils::string::startsWith(key, "invalid");
+    });
   }
 
  private:
-  processors::ListenHTTP* listen_http_ = nullptr;
-  processors::LogAttribute* log_attribute_ = nullptr;
-  std::shared_ptr<TestPlan> test_plan_;
-  std::thread thread_;
-  std::atomic_bool running_{true};
+  TestHandler handler_;
+  std::unique_ptr<TestServer> server_;
 };
 
 TEST_CASE("HTTPTestsPenalizeNoRetry", "[httptest1]") {
   using minifi::processors::InvokeHTTP;
 
   TestController testController;
-  TestHTTPServer http_server(testController);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setInfo<minifi::core::ProcessSession>();
 
@@ -120,7 +132,7 @@ TEST_CASE("InvokeHTTP fails with when flow contains invalid attribute names in H
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setDebug<InvokeHTTP>();
 
@@ -133,6 +145,7 @@ TEST_CASE("InvokeHTTP fails with when flow contains invalid attribute names in H
   auto file_contents = result.at(InvokeHTTP::RelFailure);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
+  REQUIRE(http_server.getHeaders().empty());
 }
 
 TEST_CASE("InvokeHTTP succeeds when the flow file contains an attribute that would be invalid as an HTTP header, and the policy is FAIL, but the attribute is not matched",
@@ -141,7 +154,7 @@ TEST_CASE("InvokeHTTP succeeds when the flow file contains an attribute that wou
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setDebug<InvokeHTTP>();
 
@@ -154,16 +167,16 @@ TEST_CASE("InvokeHTTP succeeds when the flow file contains an attribute that wou
   REQUIRE(result.at(InvokeHTTP::RelFailure).empty());
   const auto& success_contents = result.at(InvokeHTTP::Success);
   REQUIRE(success_contents.size() == 1);
-  REQUIRE(utils::verifyLogLinePresenceInPollTime(1s, "key:valid-header value:value2"));
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:invalid"));
+  REQUIRE(http_server.getHeaders().at("valid-header") == "value2");
+  REQUIRE(http_server.noInvalidHeaderPresent());
 }
 
 TEST_CASE("InvokeHTTP replaces invalid characters of attributes", "[httptest1]") {
   using minifi::processors::InvokeHTTP;
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setTrace<InvokeHTTP>();
 
@@ -175,16 +188,16 @@ TEST_CASE("InvokeHTTP replaces invalid characters of attributes", "[httptest1]")
   auto file_contents = result.at(InvokeHTTP::Success);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
-  REQUIRE(utils::verifyLogLinePresenceInPollTime(1s, "key:invalid-header value:value"));
-  REQUIRE(utils::verifyLogLinePresenceInPollTime(1s, "key:X-MiNiFi-Empty-Attribute-Name value:value2"));
+  REQUIRE(http_server.getHeaders().at("invalid-header") == "value");
+  REQUIRE(http_server.getHeaders().at("X-MiNiFi-Empty-Attribute-Name") == "value2");
 }
 
 TEST_CASE("InvokeHTTP drops invalid attributes from HTTP headers", "[httptest1]") {
   using minifi::processors::InvokeHTTP;
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setTrace<InvokeHTTP>();
 
@@ -197,16 +210,16 @@ TEST_CASE("InvokeHTTP drops invalid attributes from HTTP headers", "[httptest1]"
   auto file_contents = result.at(InvokeHTTP::Success);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
-  REQUIRE(utils::verifyLogLinePresenceInPollTime(1s, "key:legit-header value:value1"));
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:invalid", 0s));
+  REQUIRE(http_server.getHeaders().at("legit-header") == "value1");
+  REQUIRE(http_server.noInvalidHeaderPresent());
 }
 
 TEST_CASE("InvokeHTTP empty Attributes to Send means no attributes are sent", "[httptest1]") {
   using minifi::processors::InvokeHTTP;
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setTrace<InvokeHTTP>();
 
@@ -219,16 +232,17 @@ TEST_CASE("InvokeHTTP empty Attributes to Send means no attributes are sent", "[
   auto file_contents = result.at(InvokeHTTP::Success);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:legit-header value:value1"));
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:invalid", 0s));
+  auto header_keys = http_server.getHeaderKeys();
+  REQUIRE_FALSE(http_server.getHeaders().contains("legit-header"));
+  REQUIRE(http_server.noInvalidHeaderPresent());
 }
 
 TEST_CASE("InvokeHTTP DateHeader", "[InvokeHTTP]") {
   using minifi::processors::InvokeHTTP;
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invoke_http = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setTrace<InvokeHTTP>();
 
@@ -250,15 +264,19 @@ TEST_CASE("InvokeHTTP DateHeader", "[InvokeHTTP]") {
   auto file_contents = result.at(InvokeHTTP::Success);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
-  REQUIRE(utils::verifyEventHappenedInPollTime(1s, [&] {return LogTestController::getInstance().contains("key:Date", 0ms) == date_header;}));
+  if (date_header) {
+    REQUIRE(http_server.getHeaders().contains("Date"));
+  } else {
+    REQUIRE_FALSE(http_server.getHeaders().contains("Date"));
+  }
 }
 
 TEST_CASE("InvokeHTTP Attributes to Send uses full string matching, not substring", "[httptest1]") {
   using minifi::processors::InvokeHTTP;
 
   test::SingleProcessorTestController test_controller{std::make_unique<InvokeHTTP>("InvokeHTTP")};
   auto invokehttp = test_controller.getProcessor();
-  TestHTTPServer http_server(test_controller);
+  TestHTTPServer http_server;
 
   LogTestController::getInstance().setTrace<InvokeHTTP>();
 
@@ -271,9 +289,9 @@ TEST_CASE("InvokeHTTP Attributes to Send uses full string matching, not substrin
   auto file_contents = result.at(InvokeHTTP::Success);
   REQUIRE(file_contents.size() == 1);
   REQUIRE(test_controller.plan->getContent(file_contents[0]) == "data");
-  REQUIRE(utils::verifyLogLinePresenceInPollTime(1s, "key:header value:value2"));
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:header1 value:value1"));
-  REQUIRE_FALSE(LogTestController::getInstance().contains("key:invalid", 0s));
+  REQUIRE(http_server.getHeaders().at("header") == "value2");
+  REQUIRE_FALSE(http_server.getHeaders().contains("header1"));
+  REQUIRE(http_server.noInvalidHeaderPresent());
 }
 
 TEST_CASE("HTTPTestsResponseBodyinAttribute", "[InvokeHTTP]") {