diff --git a/docker/test/integration/features/kafka.feature b/docker/test/integration/features/kafka.feature index e7dedc2b22..f2fd08db7c 100644 --- a/docker/test/integration/features/kafka.feature +++ b/docker/test/integration/features/kafka.feature @@ -403,25 +403,6 @@ Feature: Sending data to using Kafka streaming platform using PublishKafka | 3 | 5 sec | 15 seconds | 6 | 12 | | 6 | 5 sec | 15 seconds | 12 | 24 | - Scenario Outline: Unsupported encoding attributes for ConsumeKafka throw scheduling errors - Given a ConsumeKafka processor set up in a "kafka-consumer-flow" flow - And the "" property of the ConsumeKafka processor is set to "" - And a PutFile processor with the "Directory" property set to "/tmp/output" in the "kafka-consumer-flow" flow - And the "success" relationship of the ConsumeKafka processor is connected to the PutFile - - And a kafka broker is set up in correspondence with the third-party kafka publisher - - When all instances start up - And a message with content "" is published to the "ConsumeKafkaTest" topic - And a message with content "" is published to the "ConsumeKafkaTest" topic - - Then no files are placed in the monitored directory in 20 seconds of running time - - Examples: Unsupported property values - | message 1 | message 2 | property name | property value | - | Miyamoto Musashi | Eiji Yoshikawa | Key Attribute Encoding | UTF-32 | - | Shogun | James Clavell | Message Header Encoding | UTF-32 | - Scenario: ConsumeKafka receives data via SSL Given a ConsumeKafka processor set up in a "kafka-consumer-flow" flow And these processor properties are set: diff --git a/encrypt-config/FlowConfigEncryptor.cpp b/encrypt-config/FlowConfigEncryptor.cpp index e5cb75cc0a..fa14954ae6 100644 --- a/encrypt-config/FlowConfigEncryptor.cpp +++ b/encrypt-config/FlowConfigEncryptor.cpp @@ -82,7 +82,7 @@ std::vector listSensitiveItems(const minifi::core::ProcessGroup & process_group.getAllProcessors(processors); for (const auto* processor : processors) { gsl_Expects(processor); - for (const auto& [_, property] : processor->getProperties()) { + for (const auto& [_, property] : processor->getSupportedProperties()) { if (property.isSensitive()) { sensitive_items.push_back(SensitiveItem{ .component_type = ComponentType::Processor, @@ -90,7 +90,7 @@ std::vector listSensitiveItems(const minifi::core::ProcessGroup & .component_name = processor->getName(), .item_name = property.getName(), .item_display_name = property.getDisplayName(), - .item_value = property.getValue().to_string()}); + .item_value = std::string{property.getValue().value_or("")}}); } } } @@ -104,7 +104,7 @@ std::vector listSensitiveItems(const minifi::core::ProcessGroup & continue; } processed_controller_services.insert(controller_service->getUUID()); - for (const auto& [_, property] : controller_service->getProperties()) { + for (const auto& [_, property] : controller_service->getSupportedProperties()) { if (property.isSensitive()) { sensitive_items.push_back(SensitiveItem{ .component_type = ComponentType::ControllerService, @@ -112,7 +112,7 @@ std::vector listSensitiveItems(const minifi::core::ProcessGroup & .component_name = controller_service->getName(), .item_name = property.getName(), .item_display_name = property.getDisplayName(), - .item_value = property.getValue().to_string()}); + .item_value = std::string{property.getValue().value_or("")}}); } } } diff --git a/extension-utils/include/utils/ProcessorConfigUtils.h b/extension-utils/include/utils/ProcessorConfigUtils.h index 5e5134c14f..abbf56f7d1 100644 --- a/extension-utils/include/utils/ProcessorConfigUtils.h +++ b/extension-utils/include/utils/ProcessorConfigUtils.h @@ -23,61 +23,74 @@ #include #include "core/ProcessContext.h" +#include "core/PropertyType.h" #include "utils/Enum.h" namespace org::apache::nifi::minifi::utils { -template -PropertyType getRequiredPropertyOrThrow(const core::ProcessContext& context, std::string_view property_name) { - PropertyType value; - if (!context.getProperty(property_name, value)) { - throw std::runtime_error(std::string(property_name) + " property missing or invalid"); - } - return value; -} +std::optional parseOptionalProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +std::string parseProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); + +std::optional parseOptionalBoolProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +bool parseBoolProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); + +std::optional parseOptionalU64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +uint64_t parseU64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); -std::vector listFromCommaSeparatedProperty(const core::ProcessContext& context, std::string_view property_name); -std::vector listFromRequiredCommaSeparatedProperty(const core::ProcessContext& context, std::string_view property_name); -bool parseBooleanPropertyOrThrow(const core::ProcessContext& context, std::string_view property_name); -std::chrono::milliseconds parseTimePropertyMSOrThrow(const core::ProcessContext& context, std::string_view property_name); +std::optional parseOptionalI64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +int64_t parseI64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); + +std::optional parseOptionalMsProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +std::chrono::milliseconds parseMsProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); + +std::optional parseOptionalDataSizeProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); +uint64_t parseDataSizeProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file = nullptr); template -T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { - std::string value; - if (!context.getProperty(prop.name, value)) { +T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop, const core::FlowFile* flow_file = nullptr) { + const auto enum_str = context.getProperty(prop.name, flow_file); + if (!enum_str) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' is missing"); } - auto result = magic_enum::enum_cast(value); + auto result = magic_enum::enum_cast(*enum_str); if (!result) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + value + "'"); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + *enum_str + "'"); } return result.value(); } template std::optional parseOptionalEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { - std::string value; - if (!context.getProperty(prop.name, value)) { + const auto enum_str = context.getProperty(prop.name); + + if (!enum_str) { return std::nullopt; } - auto result = magic_enum::enum_cast(value); + auto result = magic_enum::enum_cast(*enum_str); if (!result) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + value + "'"); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + *enum_str + "'"); } return result.value(); } -template -std::optional parseOptionalEnumProperty(const core::ConfigurableComponent& component, const core::PropertyReference& property) { - std::string str_value; - if (!component.getProperty(property, str_value)) { +template +std::optional> parseOptionalControllerService(const core::ProcessContext& context, const core::PropertyReference& prop, const utils::Identifier& processor_uuid) { + const auto controller_service_name = context.getProperty(prop.name); + if (!controller_service_name) { return std::nullopt; } - auto enum_value = magic_enum::enum_cast(str_value); - if (!enum_value) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, fmt::format("Property '{}' has invalid value {}", property.name, str_value)); + + const std::shared_ptr service = context.getControllerService(*controller_service_name, processor_uuid); + if (!service) { + return std::nullopt; } - return *enum_value; + + auto typed_controller_service = std::dynamic_pointer_cast(service); + if (!typed_controller_service) { + return std::nullopt; + } + + return typed_controller_service; } } // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/src/core/ProcessContext.cpp b/extension-utils/src/core/ProcessContext.cpp index 9dc9b02faa..971a6ab62c 100644 --- a/extension-utils/src/core/ProcessContext.cpp +++ b/extension-utils/src/core/ProcessContext.cpp @@ -16,21 +16,7 @@ */ #include "minifi-cpp/core/ProcessContext.h" -#include "core/TypedValues.h" namespace org::apache::nifi::minifi::core { -bool ProcessContext::getProperty(std::string_view name, detail::NotAFlowFile auto &value) const { - if constexpr (std::is_base_of_v>) { - std::string prop_str; - if (!getProperty(std::string{name}, prop_str)) { - return false; - } - value = std::decay_t(std::move(prop_str)); - return true; - } else { - return getProperty(name, value); - } -} - } // namespace org::apache::nifi::minifi::core diff --git a/extension-utils/src/utils/ListingStateManager.cpp b/extension-utils/src/utils/ListingStateManager.cpp index 4036016537..530a9cdffd 100644 --- a/extension-utils/src/utils/ListingStateManager.cpp +++ b/extension-utils/src/utils/ListingStateManager.cpp @@ -51,10 +51,11 @@ uint64_t ListingStateManager::getLatestListedKeyTimestampInMilliseconds(const st stored_listed_key_timestamp_str = it->second; } - int64_t stored_listed_key_timestamp = 0; - core::Property::StringToInt(stored_listed_key_timestamp_str, stored_listed_key_timestamp); + if (auto stored_listed_key_timestamp = parsing::parseIntegral(stored_listed_key_timestamp_str)) { + return *stored_listed_key_timestamp; + } - return stored_listed_key_timestamp; + throw std::runtime_error("Invalid listed key timestamp"); } std::unordered_set ListingStateManager::getLatestListedKeys(const std::unordered_map &state) { diff --git a/extension-utils/src/utils/ProcessorConfigUtils.cpp b/extension-utils/src/utils/ProcessorConfigUtils.cpp index 9c62453939..de13ee302f 100644 --- a/extension-utils/src/utils/ProcessorConfigUtils.cpp +++ b/extension-utils/src/utils/ProcessorConfigUtils.cpp @@ -16,37 +16,93 @@ */ #include "utils/ProcessorConfigUtils.h" -#include #include -#include -#include "range/v3/algorithm/contains.hpp" -#include "utils/StringUtils.h" +#include "utils/expected.h" namespace org::apache::nifi::minifi::utils { -std::vector listFromCommaSeparatedProperty(const core::ProcessContext& context, std::string_view property_name) { - std::string property_string; - context.getProperty(property_name, property_string); - return utils::string::splitAndTrim(property_string, ","); +std::string parseProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::expect(fmt::format("Expected valid value from {}::{}", ctx.getProcessor().getName(), property.name)); } -std::vector listFromRequiredCommaSeparatedProperty(const core::ProcessContext& context, std::string_view property_name) { - return utils::string::splitAndTrim(getRequiredPropertyOrThrow(context, property_name), ","); +std::optional parseOptionalProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + return *property_str; + } + return std::nullopt; +} + +std::optional parseOptionalBoolProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + return parsing::parseBool(*property_str) | utils::expect(fmt::format("Expected parsable bool from {}::{}", ctx.getProcessor().getName(), property.name)); + } + return std::nullopt; +} + +bool parseBoolProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::andThen(parsing::parseBool) | utils::expect(fmt::format("Expected parsable bool from {}::{}", ctx.getProcessor().getName(), property.name)); +} + +std::optional parseOptionalU64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + if (property_str->empty()) { + return std::nullopt; + } + return parsing::parseIntegral(*property_str) | utils::expect(fmt::format("Expected parsable uint64_t from {}::{}", ctx.getProcessor().getName(), property.name)); + } + + return std::nullopt; +} + +uint64_t parseU64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::andThen(parsing::parseIntegral) | utils::expect(fmt::format("Expected parsable uint64_t from {}::{}", ctx.getProcessor().getName(), property.name)); +} + +std::optional parseOptionalI64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + if (property_str->empty()) { + return std::nullopt; + } + return parsing::parseIntegral(*property_str) | utils::expect(fmt::format("Expected parsable int64_t from {}::{}", ctx.getProcessor().getName(), property.name)); + } + + return std::nullopt; } -bool parseBooleanPropertyOrThrow(const core::ProcessContext& context, std::string_view property_name) { - const std::string value_str = getRequiredPropertyOrThrow(context, property_name); - const auto maybe_value = utils::string::toBool(value_str); - if (!maybe_value) { - throw std::runtime_error(std::string(property_name) + " property is invalid: value is " + value_str); +int64_t parseI64Property(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::andThen(parsing::parseIntegral) | utils::expect(fmt::format("Expected parsable int64_t from {}::{}", ctx.getProcessor().getName(), property.name)); +} + +std::optional parseOptionalMsProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + if (property_str->empty()) { + return std::nullopt; + } + return parsing::parseDuration(*property_str) | utils::expect(fmt::format("Expected parsable duration from {}::{}", ctx.getProcessor().getName(), property.name)); } - return maybe_value.value(); + + return std::nullopt; } -std::chrono::milliseconds parseTimePropertyMSOrThrow(const core::ProcessContext& context, std::string_view property_name) { - const auto time_property = getRequiredPropertyOrThrow(context, property_name); - return time_property.getMilliseconds(); +std::chrono::milliseconds parseMsProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::andThen(parsing::parseDuration) | utils::expect(fmt::format("Expected parsable duration from {}::{}", ctx.getProcessor().getName(), property.name)); } +std::optional parseOptionalDataSizeProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + if (const auto property_str = ctx.getProperty(property, flow_file)) { + if (property_str->empty()) { + return std::nullopt; + } + return parsing::parseDataSize(*property_str) | utils::expect(fmt::format("Expected parsable data size from {}::{}", ctx.getProcessor().getName(), property.name)); + } + + return std::nullopt; +} + +uint64_t parseDataSizeProperty(const core::ProcessContext& ctx, const core::PropertyReference& property, const core::FlowFile* flow_file) { + return ctx.getProperty(property, flow_file) | utils::andThen(parsing::parseDataSize) | utils::expect(fmt::format("Expected parsable data size from {}::{}", ctx.getProcessor().getName(), property.name)); +} + + } // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/src/utils/net/Ssl.cpp b/extension-utils/src/utils/net/Ssl.cpp index 865dcb1e2d..4534c4c5f8 100644 --- a/extension-utils/src/utils/net/Ssl.cpp +++ b/extension-utils/src/utils/net/Ssl.cpp @@ -22,7 +22,7 @@ namespace org::apache::nifi::minifi::utils::net { std::optional getSslData(const core::ProcessContext& context, const core::PropertyReference& ssl_prop, const std::shared_ptr& logger) { auto getSslContextService = [&]() -> std::shared_ptr { if (auto ssl_service_name = context.getProperty(ssl_prop); ssl_service_name && !ssl_service_name->empty()) { - if (auto service = context.getControllerService(*ssl_service_name, context.getProcessorNode()->getUUID())) { + if (auto service = context.getControllerService(*ssl_service_name, context.getProcessor().getUUID())) { if (auto ssl_service = std::dynamic_pointer_cast(service)) { return ssl_service; } else { diff --git a/extensions/aws/controllerservices/AWSCredentialsService.cpp b/extensions/aws/controllerservices/AWSCredentialsService.cpp index cc4171dd19..809e0ef974 100644 --- a/extensions/aws/controllerservices/AWSCredentialsService.cpp +++ b/extensions/aws/controllerservices/AWSCredentialsService.cpp @@ -19,6 +19,7 @@ #include "AWSCredentialsService.h" #include "core/Resource.h" +#include "utils/expected.h" namespace org::apache::nifi::minifi::aws::controllers { @@ -27,19 +28,17 @@ void AWSCredentialsService::initialize() { } void AWSCredentialsService::onEnable() { - std::string value; - if (getProperty(AccessKey, value)) { - aws_credentials_provider_.setAccessKey(value); + if (const auto access_key = getProperty(AccessKey.name)) { + aws_credentials_provider_.setAccessKey(*access_key); } - if (getProperty(SecretKey, value)) { - aws_credentials_provider_.setSecretKey(value); + if (const auto secret_key = getProperty(SecretKey.name)) { + aws_credentials_provider_.setSecretKey(*secret_key); } - if (getProperty(CredentialsFile, value)) { - aws_credentials_provider_.setCredentialsFile(value); + if (const auto credentials_file = getProperty(CredentialsFile.name)) { + aws_credentials_provider_.setCredentialsFile(*credentials_file); } - bool use_default_credentials = false; - if (getProperty(UseDefaultCredentials, use_default_credentials)) { - aws_credentials_provider_.setUseDefaultCredentials(use_default_credentials); + if (const auto use_credentials = getProperty(UseDefaultCredentials.name) | minifi::utils::andThen(parsing::parseBool)) { + aws_credentials_provider_.setUseDefaultCredentials(*use_credentials); } } diff --git a/extensions/aws/controllerservices/AWSCredentialsService.h b/extensions/aws/controllerservices/AWSCredentialsService.h index f06bc6da80..62528139be 100644 --- a/extensions/aws/controllerservices/AWSCredentialsService.h +++ b/extensions/aws/controllerservices/AWSCredentialsService.h @@ -53,7 +53,7 @@ class AWSCredentialsService : public core::controller::ControllerServiceImpl { EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials") .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .isRequired(true) .build(); diff --git a/extensions/aws/processors/DeleteS3Object.cpp b/extensions/aws/processors/DeleteS3Object.cpp index 685146723d..e02efaf5b8 100644 --- a/extensions/aws/processors/DeleteS3Object.cpp +++ b/extensions/aws/processors/DeleteS3Object.cpp @@ -37,14 +37,18 @@ std::optional DeleteS3Object::buildDelet const CommonProperties &common_properties) const { gsl_Expects(client_config_); aws::s3::DeleteObjectRequestParameters params(common_properties.credentials, *client_config_); - context.getProperty(ObjectKey, params.object_key, &flow_file); + if (const auto object_key = context.getProperty(ObjectKey, &flow_file)) { + params.object_key = *object_key; + } if (params.object_key.empty() && (!flow_file.getAttribute("filename", params.object_key) || params.object_key.empty())) { logger_->log_error("No Object Key is set and default object key 'filename' attribute could not be found!"); return std::nullopt; } logger_->log_debug("DeleteS3Object: Object Key [{}]", params.object_key); - context.getProperty(Version, params.version, &flow_file); + if (const auto version = context.getProperty(Version, &flow_file)) { + params.version = *version; + } logger_->log_debug("DeleteS3Object: Version [{}]", params.version); params.bucket = common_properties.bucket; diff --git a/extensions/aws/processors/FetchS3Object.cpp b/extensions/aws/processors/FetchS3Object.cpp index 0107d14f05..194d68f542 100644 --- a/extensions/aws/processors/FetchS3Object.cpp +++ b/extensions/aws/processors/FetchS3Object.cpp @@ -26,6 +26,7 @@ #include "core/ProcessSession.h" #include "core/Resource.h" #include "utils/OptionalUtils.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::aws::processors { @@ -37,7 +38,7 @@ void FetchS3Object::initialize() { void FetchS3Object::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { S3Processor::onSchedule(context, session_factory); - context.getProperty(RequesterPays, requester_pays_); + requester_pays_ = minifi::utils::parseBoolProperty(context, RequesterPays); logger_->log_debug("FetchS3Object: RequesterPays [{}]", requester_pays_); } @@ -50,14 +51,18 @@ std::optional FetchS3Object::buildFetchS3Re get_object_params.bucket = common_properties.bucket; get_object_params.requester_pays = requester_pays_; - context.getProperty(ObjectKey, get_object_params.object_key, &flow_file); + if (const auto object_key = context.getProperty(ObjectKey, &flow_file)) { + get_object_params.object_key = * object_key; + } if (get_object_params.object_key.empty() && (!flow_file.getAttribute("filename", get_object_params.object_key) || get_object_params.object_key.empty())) { logger_->log_error("No Object Key is set and default object key 'filename' attribute could not be found!"); return std::nullopt; } logger_->log_debug("FetchS3Object: Object Key [{}]", get_object_params.object_key); - context.getProperty(Version, get_object_params.version, &flow_file); + if (const auto version = context.getProperty(Version, &flow_file)) { + get_object_params.version = *version; + } logger_->log_debug("FetchS3Object: Version [{}]", get_object_params.version); get_object_params.setClientConfig(common_properties.proxy, common_properties.endpoint_override_url); return get_object_params; diff --git a/extensions/aws/processors/FetchS3Object.h b/extensions/aws/processors/FetchS3Object.h index 3a836bde1b..6490099aac 100644 --- a/extensions/aws/processors/FetchS3Object.h +++ b/extensions/aws/processors/FetchS3Object.h @@ -51,7 +51,7 @@ class FetchS3Object : public S3Processor { .build(); EXTENSIONAPI static constexpr auto RequesterPays = core::PropertyDefinitionBuilder<>::createProperty("Requester Pays") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .withDescription("If true, indicates that the requester consents to pay any charges associated with retrieving " "objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'.") diff --git a/extensions/aws/processors/ListS3.cpp b/extensions/aws/processors/ListS3.cpp index cd9500f1f7..2ad5ad3a79 100644 --- a/extensions/aws/processors/ListS3.cpp +++ b/extensions/aws/processors/ListS3.cpp @@ -18,12 +18,13 @@ #include "ListS3.h" #include -#include #include +#include #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::aws::processors { @@ -51,29 +52,29 @@ void ListS3::onSchedule(core::ProcessContext& context, core::ProcessSessionFacto list_request_params_->setClientConfig(common_properties->proxy, common_properties->endpoint_override_url); list_request_params_->bucket = common_properties->bucket; - context.getProperty(Delimiter, list_request_params_->delimiter); + if (const auto delimiter = context.getProperty(Delimiter)) { + list_request_params_->delimiter = *delimiter; + } logger_->log_debug("ListS3: Delimiter [{}]", list_request_params_->delimiter); - context.getProperty(Prefix, list_request_params_->prefix); + if (const auto prefix = context.getProperty(Prefix)) { + list_request_params_->prefix = *prefix; + } logger_->log_debug("ListS3: Prefix [{}]", list_request_params_->prefix); - context.getProperty(UseVersions, list_request_params_->use_versions); + list_request_params_->use_versions = minifi::utils::parseBoolProperty(context, UseVersions); logger_->log_debug("ListS3: UseVersions [{}]", list_request_params_->use_versions); - if (auto minimum_object_age = context.getProperty(MinimumObjectAge)) { - list_request_params_->min_object_age = minimum_object_age->getMilliseconds().count(); - } else { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Minimum Object Age missing or invalid"); - } + list_request_params_->min_object_age = minifi::utils::parseMsProperty(context, MinimumObjectAge).count(); logger_->log_debug("S3Processor: Minimum Object Age [{}]", list_request_params_->min_object_age); - context.getProperty(WriteObjectTags, write_object_tags_); + write_object_tags_ = minifi::utils::parseBoolProperty(context, WriteObjectTags); logger_->log_debug("ListS3: WriteObjectTags [{}]", write_object_tags_); - context.getProperty(WriteUserMetadata, write_user_metadata_); + write_user_metadata_ = minifi::utils::parseBoolProperty(context, WriteUserMetadata); logger_->log_debug("ListS3: WriteUserMetadata [{}]", write_user_metadata_); - context.getProperty(RequesterPays, requester_pays_); + requester_pays_ = minifi::utils::parseBoolProperty(context, RequesterPays); logger_->log_debug("ListS3: RequesterPays [{}]", requester_pays_); } diff --git a/extensions/aws/processors/ListS3.h b/extensions/aws/processors/ListS3.h index fd69fe7544..9f03acc98a 100644 --- a/extensions/aws/processors/ListS3.h +++ b/extensions/aws/processors/ListS3.h @@ -45,30 +45,30 @@ class ListS3 : public S3Processor { EXTENSIONAPI static constexpr auto UseVersions = core::PropertyDefinitionBuilder<>::createProperty("Use Versions") .withDescription("Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto MinimumObjectAge = core::PropertyDefinitionBuilder<>::createProperty("Minimum Object Age") .withDescription("The minimum age that an S3 object must be in order to be considered; any object younger than this amount of time (according to last modification date) will be ignored.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("0 sec") .build(); EXTENSIONAPI static constexpr auto WriteObjectTags = core::PropertyDefinitionBuilder<>::createProperty("Write Object Tags") .withDescription("If set to 'true', the tags associated with the S3 object will be written as FlowFile attributes.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto WriteUserMetadata = core::PropertyDefinitionBuilder<>::createProperty("Write User Metadata") .isRequired(true) .withDescription("If set to 'true', the user defined metadata associated with the S3 object will be added to FlowFile attributes/records.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto RequesterPays = core::PropertyDefinitionBuilder<>::createProperty("Requester Pays") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .withDescription("If true, indicates that the requester consents to pay any charges associated with listing the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. " "Note that this setting is only used if Write User Metadata is true.") diff --git a/extensions/aws/processors/PutS3Object.cpp b/extensions/aws/processors/PutS3Object.cpp index 2415ad72b6..2e9413c951 100644 --- a/extensions/aws/processors/PutS3Object.cpp +++ b/extensions/aws/processors/PutS3Object.cpp @@ -41,15 +41,14 @@ void PutS3Object::fillUserMetadata(core::ProcessContext& context) { const auto &dynamic_prop_keys = context.getDynamicPropertyKeys(); bool first_property = true; for (const auto &prop_key : dynamic_prop_keys) { - std::string prop_value; - if (context.getDynamicProperty(prop_key, prop_value) && !prop_value.empty()) { - logger_->log_debug("PutS3Object: DynamicProperty: [{}] -> [{}]", prop_key, prop_value); - user_metadata_map_.emplace(prop_key, prop_value); + if (const auto prop_value = context.getDynamicProperty(prop_key); prop_value && !prop_value->empty()) { + logger_->log_debug("PutS3Object: DynamicProperty: [{}] -> [{}]", prop_key, *prop_value); + user_metadata_map_.emplace(prop_key, *prop_value); if (first_property) { - user_metadata_ = minifi::utils::string::join_pack(prop_key, "=", prop_value); + user_metadata_ = minifi::utils::string::join_pack(prop_key, "=", *prop_value); first_property = false; } else { - user_metadata_ += minifi::utils::string::join_pack(",", prop_key, "=", prop_value); + user_metadata_ += minifi::utils::string::join_pack(",", prop_key, "=", *prop_value); } } } @@ -59,38 +58,34 @@ void PutS3Object::fillUserMetadata(core::ProcessContext& context) { void PutS3Object::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { S3Processor::onSchedule(context, session_factory); - if (!context.getProperty(StorageClass, storage_class_) - || storage_class_.empty() - || !ranges::contains(STORAGE_CLASSES, storage_class_)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Storage Class property missing or invalid"); - } + storage_class_ = minifi::utils::parseProperty(context, StorageClass); logger_->log_debug("PutS3Object: Storage Class [{}]", storage_class_); - if (!context.getProperty(ServerSideEncryption, server_side_encryption_) - || server_side_encryption_.empty() - || !ranges::contains(SERVER_SIDE_ENCRYPTIONS, server_side_encryption_)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Server Side Encryption property missing or invalid"); - } + server_side_encryption_ = minifi::utils::parseProperty(context, ServerSideEncryption); logger_->log_debug("PutS3Object: Server Side Encryption [{}]", server_side_encryption_); - if (auto use_path_style_access = context.getProperty(UsePathStyleAccess)) { + if (const auto use_path_style_access = minifi::utils::parseOptionalBoolProperty(context, UsePathStyleAccess)) { use_virtual_addressing_ = !*use_path_style_access; } - if (!context.getProperty(MultipartThreshold, multipart_threshold_) || multipart_threshold_ > getMaxUploadSize() || multipart_threshold_ < getMinPartSize()) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Multipart Threshold is not between the valid 5MB and 5GB range!"); - } + multipart_threshold_ = context.getProperty(MultipartThreshold) + | minifi::utils::andThen([&](const auto str) { return parsing::parseDataSizeMinMax(str, getMinPartSize(), getMaxUploadSize()); }) + | minifi::utils::expect("Multipart Part Size is not between the valid 5MB and 5GB range!"); + logger_->log_debug("PutS3Object: Multipart Threshold {}", multipart_threshold_); - if (!context.getProperty(MultipartPartSize, multipart_size_) || multipart_size_ > getMaxUploadSize() || multipart_size_ < getMinPartSize()) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Multipart Part Size is not between the valid 5MB and 5GB range!"); - } + + multipart_size_ = context.getProperty(MultipartPartSize) + | minifi::utils::andThen([&](const auto str) { return parsing::parseDataSizeMinMax(str, getMinPartSize(), getMaxUploadSize()); }) + | minifi::utils::expect("Multipart Part Size is not between the valid 5MB and 5GB range!"); + + logger_->log_debug("PutS3Object: Multipart Size {}", multipart_size_); - multipart_upload_ageoff_interval_ = minifi::utils::getRequiredPropertyOrThrow(context, MultipartUploadAgeOffInterval.name).getMilliseconds(); + multipart_upload_ageoff_interval_ = minifi::utils::parseMsProperty(context, MultipartUploadAgeOffInterval); logger_->log_debug("PutS3Object: Multipart Upload Ageoff Interval {}", multipart_upload_ageoff_interval_); - multipart_upload_max_age_threshold_ = minifi::utils::getRequiredPropertyOrThrow(context, MultipartUploadMaxAgeThreshold.name).getMilliseconds(); + multipart_upload_max_age_threshold_ = minifi::utils::parseMsProperty(context, MultipartUploadMaxAgeThreshold); logger_->log_debug("PutS3Object: Multipart Upload Max Age Threshold {}", multipart_upload_max_age_threshold_); fillUserMetadata(context); @@ -119,7 +114,10 @@ bool PutS3Object::setCannedAcl( core::ProcessContext& context, const core::FlowFile& flow_file, aws::s3::PutObjectRequestParameters &put_s3_request_params) const { - context.getProperty(CannedACL, put_s3_request_params.canned_acl, &flow_file); + if (const auto canned_acl = context.getProperty(CannedACL, &flow_file)) { + put_s3_request_params.canned_acl = *canned_acl; + } + if (!put_s3_request_params.canned_acl.empty() && !ranges::contains(CANNED_ACLS, put_s3_request_params.canned_acl)) { logger_->log_error("Canned ACL is invalid!"); return false; @@ -132,22 +130,21 @@ bool PutS3Object::setAccessControl( core::ProcessContext& context, const core::FlowFile& flow_file, aws::s3::PutObjectRequestParameters &put_s3_request_params) const { - std::string value; - if (context.getProperty(FullControlUserList, value, &flow_file) && !value.empty()) { - put_s3_request_params.fullcontrol_user_list = parseAccessControlList(value); - logger_->log_debug("PutS3Object: Full Control User List [{}]", value); + if (const auto full_control_user_list = context.getProperty(FullControlUserList, &flow_file)) { + put_s3_request_params.fullcontrol_user_list = parseAccessControlList(*full_control_user_list); + logger_->log_debug("PutS3Object: Full Control User List [{}]", *full_control_user_list); } - if (context.getProperty(ReadPermissionUserList, value, &flow_file) && !value.empty()) { - put_s3_request_params.read_permission_user_list = parseAccessControlList(value); - logger_->log_debug("PutS3Object: Read Permission User List [{}]", value); + if (const auto read_permission_user_list = context.getProperty(ReadPermissionUserList, &flow_file)) { + put_s3_request_params.read_permission_user_list = parseAccessControlList(*read_permission_user_list); + logger_->log_debug("PutS3Object: Read Permission User List [{}]", *read_permission_user_list); } - if (context.getProperty(ReadACLUserList, value, &flow_file) && !value.empty()) { - put_s3_request_params.read_acl_user_list = parseAccessControlList(value); - logger_->log_debug("PutS3Object: Read ACL User List [{}]", value); + if (const auto read_acl_user_list = context.getProperty(ReadACLUserList, &flow_file)) { + put_s3_request_params.read_acl_user_list = parseAccessControlList(*read_acl_user_list); + logger_->log_debug("PutS3Object: Read ACL User List [{}]", *read_acl_user_list); } - if (context.getProperty(WriteACLUserList, value, &flow_file) && !value.empty()) { - put_s3_request_params.write_acl_user_list = parseAccessControlList(value); - logger_->log_debug("PutS3Object: Write ACL User List [{}]", value); + if (const auto write_acl_user_list = context.getProperty(WriteACLUserList, &flow_file)) { + put_s3_request_params.write_acl_user_list = parseAccessControlList(*write_acl_user_list); + logger_->log_debug("PutS3Object: Write ACL User List [{}]", *write_acl_user_list); } return setCannedAcl(context, flow_file, put_s3_request_params); @@ -165,14 +162,14 @@ std::optional PutS3Object::buildPutS3Reques params.server_side_encryption = server_side_encryption_; params.storage_class = storage_class_; - context.getProperty(ObjectKey, params.object_key, &flow_file); + params.object_key = context.getProperty(ObjectKey, &flow_file).value_or(""); if (params.object_key.empty() && (!flow_file.getAttribute("filename", params.object_key) || params.object_key.empty())) { logger_->log_error("No Object Key is set and default object key 'filename' attribute could not be found!"); return std::nullopt; } logger_->log_debug("PutS3Object: Object Key [{}]", params.object_key); - context.getProperty(ContentType, params.content_type, &flow_file); + params.content_type = context.getProperty(ContentType, &flow_file).value_or(""); logger_->log_debug("PutS3Object: Content Type [{}]", params.content_type); if (!setAccessControl(context, flow_file, params)) { diff --git a/extensions/aws/processors/PutS3Object.h b/extensions/aws/processors/PutS3Object.h index 849de3137c..c9427e856a 100644 --- a/extensions/aws/processors/PutS3Object.h +++ b/extensions/aws/processors/PutS3Object.h @@ -84,20 +84,24 @@ class PutS3Object : public S3Processor { EXTENSIONAPI static constexpr auto FullControlUserList = core::PropertyDefinitionBuilder<>::createProperty("FullControl User List") .withDescription("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have Full Control for an object.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto ReadPermissionUserList = core::PropertyDefinitionBuilder<>::createProperty("Read Permission User List") .withDescription("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have Read Access for an object.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto ReadACLUserList = core::PropertyDefinitionBuilder<>::createProperty("Read ACL User List") .withDescription("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to read " "the Access Control List for an object.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto WriteACLUserList = core::PropertyDefinitionBuilder<>::createProperty("Write ACL User List") .withDescription("A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to change " "the Access Control List for an object.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto CannedACL = core::PropertyDefinitionBuilder<>::createProperty("Canned ACL") .withDescription("Amazon Canned ACL for an object. Allowed values: BucketOwnerFullControl, BucketOwnerRead, AuthenticatedRead, " @@ -107,34 +111,34 @@ class PutS3Object : public S3Processor { EXTENSIONAPI static constexpr auto UsePathStyleAccess = core::PropertyDefinitionBuilder<>::createProperty("Use Path Style Access") .withDescription("Path-style access can be enforced by setting this property to true. Set it to true if your endpoint does not support " "virtual-hosted-style requests, only path-style requests.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartThreshold = core::PropertyDefinitionBuilder<>::createProperty("Multipart Threshold") .withDescription("Specifies the file size threshold for switch from the PutS3Object API to the PutS3MultipartUpload API. " "Flow files bigger than this limit will be sent using the multipart process. The valid range is 5MB to 5GB.") - .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withValidator(core::StandardPropertyTypes::DATA_SIZE_VALIDATOR) .withDefaultValue("5 GB") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartPartSize = core::PropertyDefinitionBuilder<>::createProperty("Multipart Part Size") .withDescription("Specifies the part size for use when the PutS3Multipart Upload API is used. " "Flow files will be broken into chunks of this size for the upload process, but the last part sent can be smaller since it is not padded. The valid range is 5MB to 5GB.") - .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withValidator(core::StandardPropertyTypes::DATA_SIZE_VALIDATOR) .withDefaultValue("5 GB") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartUploadAgeOffInterval = core::PropertyDefinitionBuilder<>::createProperty("Multipart Upload AgeOff Interval") .withDescription("Specifies the interval at which existing multipart uploads in AWS S3 will be evaluated for ageoff. " "When processor is triggered it will initiate the ageoff evaluation if this interval has been exceeded.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("60 min") .isRequired(true) .build(); EXTENSIONAPI static constexpr auto MultipartUploadMaxAgeThreshold = core::PropertyDefinitionBuilder<>::createProperty("Multipart Upload Max Age Threshold") .withDescription("Specifies the maximum age for existing multipart uploads in AWS S3. When the ageoff process occurs, any upload older than this threshold will be aborted.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("7 days") .isRequired(true) .build(); diff --git a/extensions/aws/processors/S3Processor.cpp b/extensions/aws/processors/S3Processor.cpp index eb2b2c528c..ff76a548a8 100644 --- a/extensions/aws/processors/S3Processor.cpp +++ b/extensions/aws/processors/S3Processor.cpp @@ -17,17 +17,18 @@ #include "S3Processor.h" -#include #include +#include #include -#include "core/ProcessContext.h" -#include "S3Wrapper.h" #include "AWSCredentialsService.h" +#include "S3Wrapper.h" +#include "core/ProcessContext.h" #include "properties/Properties.h" #include "range/v3/algorithm/contains.hpp" -#include "utils/StringUtils.h" #include "utils/HTTPUtils.h" +#include "utils/StringUtils.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::aws::processors { @@ -43,24 +44,12 @@ S3Processor::S3Processor(std::string_view name, const minifi::utils::Identifier& } std::optional S3Processor::getAWSCredentialsFromControllerService(core::ProcessContext& context) const { - std::string service_name; - if (!context.getProperty(AWSCredentialsProviderService, service_name) || service_name.empty()) { - return std::nullopt; + if (const auto aws_credentials_service = minifi::utils::parseOptionalControllerService(context, AWSCredentialsProviderService, getUUID())) { + return (*aws_credentials_service)->getAWSCredentials(); } + logger_->log_error("AWS credentials service could not be found"); - std::shared_ptr service = context.getControllerService(service_name, getUUID()); - if (!service) { - logger_->log_error("AWS credentials service with name: '{}' could not be found", service_name); - return std::nullopt; - } - - auto aws_credentials_service = std::dynamic_pointer_cast(service); - if (!aws_credentials_service) { - logger_->log_error("Controller service with name: '{}' is not an AWS credentials service", service_name); - return std::nullopt; - } - - return aws_credentials_service->getAWSCredentials(); + return std::nullopt; } std::optional S3Processor::getAWSCredentials( @@ -73,19 +62,17 @@ std::optional S3Processor::getAWSCredentials( } aws::AWSCredentialsProvider aws_credentials_provider; - std::string value; - if (context.getProperty(AccessKey, value, flow_file)) { - aws_credentials_provider.setAccessKey(value); + if (const auto access_key = context.getProperty(AccessKey.name, flow_file)) { + aws_credentials_provider.setAccessKey(*access_key); } - if (context.getProperty(SecretKey, value, flow_file)) { - aws_credentials_provider.setSecretKey(value); + if (const auto secret_key = context.getProperty(SecretKey.name, flow_file)) { + aws_credentials_provider.setSecretKey(*secret_key); } - if (context.getProperty(CredentialsFile, value)) { - aws_credentials_provider.setCredentialsFile(value); + if (const auto credentials_file = context.getProperty(CredentialsFile.name, flow_file)) { + aws_credentials_provider.setCredentialsFile(*credentials_file); } - bool use_default_credentials = false; - if (context.getProperty(UseDefaultCredentials, use_default_credentials)) { - aws_credentials_provider.setUseDefaultCredentials(use_default_credentials); + if (const auto use_credentials = context.getProperty(UseDefaultCredentials.name, flow_file) | minifi::utils::andThen(parsing::parseBool)) { + aws_credentials_provider.setUseDefaultCredentials(*use_credentials); } return aws_credentials_provider.getAWSCredentials(); @@ -93,14 +80,12 @@ std::optional S3Processor::getAWSCredentials( std::optional S3Processor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) { aws::s3::ProxyOptions proxy; - context.getProperty(ProxyHost, proxy.host, flow_file); - std::string port_str; - if (context.getProperty(ProxyPort, port_str, flow_file) && !port_str.empty() && !core::Property::StringToInt(port_str, proxy.port)) { - logger_->log_error("Proxy port invalid"); - return std::nullopt; - } - context.getProperty(ProxyUsername, proxy.username, flow_file); - context.getProperty(ProxyPassword, proxy.password, flow_file); + + proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); + proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); + proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or(""); + proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or(""); + if (!proxy.host.empty()) { logger_->log_info("Proxy for S3Processor was set."); } @@ -109,19 +94,16 @@ std::optional S3Processor::getProxy(core::ProcessContext& void S3Processor::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { client_config_ = Aws::Client::ClientConfiguration(); - std::string value; - if (!context.getProperty(Bucket, value) || value.empty()) { + if (!getProperty(Bucket.name)) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Bucket property missing or invalid"); } - if (!context.getProperty(Region, client_config_->region) || client_config_->region.empty() || !ranges::contains(region::REGIONS, client_config_->region)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Region property missing or invalid"); - } + client_config_->region = context.getProperty(Region) | minifi::utils::expect("Region property missing or invalid"); logger_->log_debug("S3Processor: Region [{}]", client_config_->region); - if (auto communications_timeout = context.getProperty(CommunicationsTimeout)) { - logger_->log_debug("S3Processor: Communications Timeout {}", communications_timeout->getMilliseconds()); - client_config_->connectTimeoutMs = gsl::narrow(communications_timeout->getMilliseconds().count()); // NOLINT(runtime/int,google-runtime-int) + if (auto communications_timeout = minifi::utils::parseOptionalMsProperty(context, CommunicationsTimeout)) { + logger_->log_debug("S3Processor: Communications Timeout {}", *communications_timeout); + client_config_->connectTimeoutMs = gsl::narrow(communications_timeout->count()); // NOLINT(runtime/int,google-runtime-int) } else { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Communications Timeout missing or invalid"); } @@ -136,9 +118,11 @@ std::optional S3Processor::getCommonELSupportedProperties( core::ProcessContext& context, const core::FlowFile* const flow_file) { CommonProperties properties; - if (!context.getProperty(Bucket, properties.bucket, flow_file) || properties.bucket.empty()) { + if (auto bucket = context.getProperty(Bucket, flow_file); !bucket || bucket->empty()) { logger_->log_error("Bucket '{}' is invalid or empty!", properties.bucket); return std::nullopt; + } else { + properties.bucket = *bucket; } logger_->log_debug("S3Processor: Bucket [{}]", properties.bucket); @@ -155,8 +139,9 @@ std::optional S3Processor::getCommonELSupportedProperties( } properties.proxy = proxy.value(); - context.getProperty(EndpointOverrideURL, properties.endpoint_override_url, flow_file); - if (!properties.endpoint_override_url.empty()) { + const auto endpoint_override_url = context.getProperty(EndpointOverrideURL, flow_file); + if (endpoint_override_url) { + properties.endpoint_override_url = *endpoint_override_url; logger_->log_debug("S3Processor: Endpoint Override URL [{}]", properties.endpoint_override_url); } diff --git a/extensions/aws/processors/S3Processor.h b/extensions/aws/processors/S3Processor.h index df273b3433..3525e80429 100644 --- a/extensions/aws/processors/S3Processor.h +++ b/extensions/aws/processors/S3Processor.h @@ -100,6 +100,7 @@ class S3Processor : public core::ProcessorImpl { EXTENSIONAPI static constexpr auto Bucket = core::PropertyDefinitionBuilder<>::createProperty("Bucket") .withDescription("The S3 bucket") .isRequired(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto AccessKey = core::PropertyDefinitionBuilder<>::createProperty("Access Key") @@ -125,7 +126,7 @@ class S3Processor : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto CommunicationsTimeout = core::PropertyDefinitionBuilder<>::createProperty("Communications Timeout") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("30 sec") .withDescription("Sets the timeout of the communication between the AWS server and the client") .build(); @@ -155,7 +156,7 @@ class S3Processor : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials") .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .isRequired(true) .build(); diff --git a/extensions/aws/s3/MultipartUploadStateStorage.cpp b/extensions/aws/s3/MultipartUploadStateStorage.cpp index 2e9a79d5e0..076245dd8a 100644 --- a/extensions/aws/s3/MultipartUploadStateStorage.cpp +++ b/extensions/aws/s3/MultipartUploadStateStorage.cpp @@ -57,11 +57,11 @@ std::optional MultipartUploadStateStorage::getState(const MultipartUploadState state; state.upload_id = state_map[state_key + ".upload_id"]; - core::Property::StringToInt(state_map[state_key + ".upload_time"], state.upload_time_ms_since_epoch); - core::Property::StringToInt(state_map[state_key + ".uploaded_parts"], state.uploaded_parts); - core::Property::StringToInt(state_map[state_key + ".uploaded_size"], state.uploaded_size); - core::Property::StringToInt(state_map[state_key + ".part_size"], state.part_size); - core::Property::StringToInt(state_map[state_key + ".full_size"], state.full_size); + state.upload_time_ms_since_epoch = parsing::parseIntegral(state_map[state_key + ".upload_time"]) | utils::expect("Expected parsable upload_time_ms_since_epoch"); + state.uploaded_parts = parsing::parseIntegral(state_map[state_key + ".uploaded_parts"]) | utils::expect("Expected parsable state.uploaded_parts"); + state.uploaded_size = parsing::parseIntegral(state_map[state_key + ".uploaded_size"]) | utils::expect("Expected parsable state.uploaded_size"); + state.part_size = parsing::parseIntegral(state_map[state_key + ".part_size"]) | utils::expect("Expected parsable state.part_size"); + state.full_size = parsing::parseIntegral(state_map[state_key + ".full_size"]) | utils::expect("Expected parsable state.full_size"); state.uploaded_etags = minifi::utils::string::splitAndTrimRemovingEmpty(state_map[state_key + ".uploaded_etags"], ";"); return state; } @@ -110,15 +110,15 @@ void MultipartUploadStateStorage::removeAgedStates(std::chrono::milliseconds mul if (!minifi::utils::string::endsWith(property_key, upload_time_suffix)) { continue; } - int64_t stored_upload_time{}; - if (!core::Property::StringToInt(value, stored_upload_time)) { + if (const auto stored_upload_time = parsing::parseIntegral(value); !stored_upload_time) { logger_->log_error("Multipart upload cache key '{}' has invalid value '{}'", property_key, value); continue; - } - auto upload_time = Aws::Utils::DateTime(stored_upload_time); - if (upload_time < age_off_time) { - auto state_key_and_property_name = property_key.substr(0, property_key.size() - upload_time_suffix.size()); - keys_to_remove.push_back(state_key_and_property_name); + } else { + auto upload_time = Aws::Utils::DateTime(*stored_upload_time); + if (upload_time < age_off_time) { + auto state_key_and_property_name = property_key.substr(0, property_key.size() - upload_time_suffix.size()); + keys_to_remove.push_back(state_key_and_property_name); + } } } for (const auto& key : keys_to_remove) { diff --git a/extensions/aws/tests/DeleteS3ObjectTests.cpp b/extensions/aws/tests/DeleteS3ObjectTests.cpp index f75f3a4aa2..bf06e060a2 100644 --- a/extensions/aws/tests/DeleteS3ObjectTests.cpp +++ b/extensions/aws/tests/DeleteS3ObjectTests.cpp @@ -74,14 +74,14 @@ TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test required property not set", " plan->setDynamicProperty(update_attribute, "filename", ""); } - SECTION("Test region is empty") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Region", ""); - } - REQUIRE_THROWS_AS(test_controller.runSession(plan, true), minifi::Exception); } +TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Non blank validator tests") { + setRequiredProperties(); + CHECK_FALSE(plan->setProperty(s3_processor, "Region", "")); +} + TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); setProxy(); diff --git a/extensions/aws/tests/FetchS3ObjectTests.cpp b/extensions/aws/tests/FetchS3ObjectTests.cpp index eeaaa1c4b3..5920c13c57 100644 --- a/extensions/aws/tests/FetchS3ObjectTests.cpp +++ b/extensions/aws/tests/FetchS3ObjectTests.cpp @@ -91,14 +91,14 @@ TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Test required property not set", "[ plan->setDynamicProperty(update_attribute, "filename", ""); } - SECTION("Test region is empty") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Region", ""); - } - REQUIRE_THROWS_AS(test_controller.runSession(plan, true), minifi::Exception); } +TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Non blank validator tests") { + setRequiredProperties(); + CHECK_FALSE(plan->setProperty(s3_processor, "Region", "")); +} + TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); setProxy(); diff --git a/extensions/aws/tests/ListS3Tests.cpp b/extensions/aws/tests/ListS3Tests.cpp index 1792f23754..424e83fd53 100644 --- a/extensions/aws/tests/ListS3Tests.cpp +++ b/extensions/aws/tests/ListS3Tests.cpp @@ -64,14 +64,14 @@ TEST_CASE_METHOD(ListS3TestsFixture, "Test required property not set", "[awsS3Er setAccesKeyCredentialsInProcessor(); } - SECTION("Test region is empty") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Region", ""); - } - REQUIRE_THROWS_AS(test_controller.runSession(plan, true), minifi::Exception); } +TEST_CASE_METHOD(ListS3TestsFixture, "Non blank validator tests") { + setRequiredProperties(); + CHECK_FALSE(plan->setProperty(s3_processor, "Region", "")); +} + TEST_CASE_METHOD(ListS3TestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); setProxy(); diff --git a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp index 29458b2301..348e42a590 100644 --- a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp +++ b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp @@ -116,8 +116,8 @@ TEST_CASE_METHOD(MultipartUploadStateStorageTestFixture, "Remove aged off state" state2.uploaded_etags = {"etag4", "etag5"}; upload_storage_->storeState("test_bucket", "key2", state2); upload_storage_->removeAgedStates(10min); - REQUIRE(upload_storage_->getState("test_bucket", "key1") == std::nullopt); - REQUIRE(upload_storage_->getState("test_bucket", "key2") == state2); + CHECK(upload_storage_->getState("test_bucket", "key1") == std::nullopt); + CHECK(upload_storage_->getState("test_bucket", "key2") == state2); } } // namespace org::apache::nifi::minifi::test diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp index 4477dc98f5..246de77a96 100644 --- a/extensions/aws/tests/PutS3ObjectTests.cpp +++ b/extensions/aws/tests/PutS3ObjectTests.cpp @@ -102,27 +102,18 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test required property not set", "[aw SECTION("Test no object key is set") { setRequiredProperties(); - plan->setDynamicProperty(update_attribute, "filename", ""); - } - - SECTION("Test storage class is empty") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Storage Class", ""); - } - - SECTION("Test region is empty") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Region", ""); - } - - SECTION("Test no server side encryption is set") { - setRequiredProperties(); - plan->setProperty(s3_processor, "Server Side Encryption", ""); + CHECK(plan->setDynamicProperty(update_attribute, "filename", "")); } REQUIRE_THROWS_AS(test_controller.runSession(plan), minifi::Exception); } +TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Non blank properties", "[awsS3Config]") { + setRequiredProperties(); + CHECK_FALSE(plan->setProperty(s3_processor, "Server Side Encryption", "")); + CHECK_FALSE(plan->setProperty(s3_processor, "Storage Class", "")); +} + TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test incomplete credentials in credentials service", "[awsS3Config]") { setBucket(); plan->setProperty(aws_credentials_service, "Secret Key", "secret"); @@ -261,7 +252,7 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test multipart upload limits", "[awsS plan->setProperty(s3_processor, "Multipart Part Size", "51 GB"); } - REQUIRE_THROWS_AS(test_controller.runSession(plan), minifi::Exception); + REQUIRE_THROWS_AS(test_controller.runSession(plan), std::runtime_error); } TEST_CASE_METHOD(PutS3ObjectUploadLimitChangedTestsFixture, "Test multipart upload", "[awsS3MultipartUpload]") { diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp b/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp index bd21e39a40..4d0ae2b394 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.cpp @@ -29,25 +29,23 @@ void AzureStorageCredentialsService::initialize() { } void AzureStorageCredentialsService::onEnable() { - std::string value; - if (getProperty(StorageAccountName, value)) { - credentials_.setStorageAccountName(value); + if (auto storage_account_name = getProperty(StorageAccountName.name)) { + credentials_.setStorageAccountName(*storage_account_name); } - if (getProperty(StorageAccountKey, value)) { - credentials_.setStorageAccountKey(value); + if (auto storage_account_key = getProperty(StorageAccountKey.name)) { + credentials_.setStorageAccountKey(*storage_account_key); } - if (getProperty(SASToken, value)) { - credentials_.setSasToken(value); + if (auto sas_token = getProperty(SASToken.name)) { + credentials_.setSasToken(*sas_token); } - if (getProperty(CommonStorageAccountEndpointSuffix, value)) { - credentials_.setEndpontSuffix(value); + if (auto common_storage_account_endpoint_suffix = getProperty(CommonStorageAccountEndpointSuffix.name)) { + credentials_.setEndpontSuffix(*common_storage_account_endpoint_suffix); } - if (getProperty(ConnectionString, value)) { - credentials_.setConnectionString(value); + if (auto connection_String = getProperty(ConnectionString.name)) { + credentials_.setConnectionString(*connection_String); } - bool use_managed_identity_credentials = false; - if (getProperty(UseManagedIdentityCredentials, use_managed_identity_credentials)) { - credentials_.setUseManagedIdentityCredentials(use_managed_identity_credentials); + if (auto use_managed_identity_credentials = getProperty(UseManagedIdentityCredentials.name) | utils::andThen(parsing::parseBool)) { + credentials_.setUseManagedIdentityCredentials(*use_managed_identity_credentials); } } diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.h b/extensions/azure/controllerservices/AzureStorageCredentialsService.h index 5be6a6f096..7fe6046f07 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.h +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.h @@ -59,7 +59,7 @@ class AzureStorageCredentialsService : public core::controller::ControllerServic EXTENSIONAPI static constexpr auto UseManagedIdentityCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Managed Identity Credentials") .withDescription("If true Managed Identity credentials will be used together with the Storage Account Name for authentication.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ diff --git a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp index 0f403017dc..d3ec978e62 100644 --- a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp @@ -21,44 +21,42 @@ #include "AzureBlobStorageProcessorBase.h" #include "core/ProcessContext.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::azure::processors { void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - std::string value; - if (!context.getProperty(ContainerName, value) || value.empty()) { + if (!getProperty(ContainerName.name)) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Container Name property missing or invalid"); } - if (context.getProperty(AzureStorageCredentialsService, value) && !value.empty()) { - logger_->log_info("Getting Azure Storage credentials from controller service with name: '{}'", value); + if (auto azure_storage_credentials_service = getProperty(AzureStorageCredentialsService.name)) { + logger_->log_info("Getting Azure Storage credentials from controller service with name: '{}'", *azure_storage_credentials_service); return; } - if (!context.getProperty(UseManagedIdentityCredentials, use_managed_identity_credentials_)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Use Managed Identity Credentials is invalid."); - } + use_managed_identity_credentials_ = minifi::utils::parseBoolProperty(context, UseManagedIdentityCredentials); if (use_managed_identity_credentials_) { logger_->log_info("Using Managed Identity for authentication"); return; } - if (context.getProperty(ConnectionString, value) && !value.empty()) { + if (auto connection_string = getProperty(ConnectionString.name); connection_string && !connection_string->empty()) { logger_->log_info("Using connectionstring directly for Azure Storage authentication"); return; } - if (!context.getProperty(StorageAccountName, value) || value.empty()) { + if (!getProperty(StorageAccountName.name)) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Storage Account Name property missing or invalid"); } - if (context.getProperty(StorageAccountKey, value) && !value.empty()) { + if (const auto storage_account_key = getProperty(StorageAccountKey.name); storage_account_key && !storage_account_key->empty()) { logger_->log_info("Using storage account name and key for authentication"); return; } - if (!context.getProperty(SASToken, value) || value.empty()) { + if (const auto sas_token = getProperty(SASToken.name); !sas_token || sas_token->empty()) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Neither Storage Account Key nor SAS Token property was set."); } @@ -68,21 +66,20 @@ void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, co storage::AzureStorageCredentials AzureBlobStorageProcessorBase::getAzureCredentialsFromProperties( core::ProcessContext &context, const core::FlowFile* const flow_file) const { storage::AzureStorageCredentials credentials; - std::string value; - if (context.getProperty(StorageAccountName, value, flow_file)) { - credentials.setStorageAccountName(value); + if (const auto value = context.getProperty(StorageAccountName, flow_file)) { + credentials.setStorageAccountName(*value); } - if (context.getProperty(StorageAccountKey, value, flow_file)) { - credentials.setStorageAccountKey(value); + if (const auto value = context.getProperty(StorageAccountKey, flow_file)) { + credentials.setStorageAccountKey(*value); } - if (context.getProperty(SASToken, value, flow_file)) { - credentials.setSasToken(value); + if (const auto value = context.getProperty(SASToken, flow_file)) { + credentials.setSasToken(*value); } - if (context.getProperty(CommonStorageAccountEndpointSuffix, value, flow_file)) { - credentials.setEndpontSuffix(value); + if (const auto value = context.getProperty(CommonStorageAccountEndpointSuffix, flow_file)) { + credentials.setEndpontSuffix(*value); } - if (context.getProperty(ConnectionString, value, flow_file)) { - credentials.setConnectionString(value); + if (const auto value = context.getProperty(ConnectionString, flow_file)) { + credentials.setConnectionString(*value); } credentials.setUseManagedIdentityCredentials(use_managed_identity_credentials_); return credentials; @@ -99,7 +96,8 @@ bool AzureBlobStorageProcessorBase::setCommonStorageParameters( params.credentials = *credentials; - if (!context.getProperty(ContainerName, params.container_name, flow_file) || params.container_name.empty()) { + params.container_name = context.getProperty(ContainerName, flow_file).value_or(""); + if (params.container_name.empty()) { logger_->log_error("Container Name is invalid or empty!"); return false; } diff --git a/extensions/azure/processors/AzureBlobStorageProcessorBase.h b/extensions/azure/processors/AzureBlobStorageProcessorBase.h index 822f2bac01..9b834fbd7b 100644 --- a/extensions/azure/processors/AzureBlobStorageProcessorBase.h +++ b/extensions/azure/processors/AzureBlobStorageProcessorBase.h @@ -41,6 +41,7 @@ class AzureBlobStorageProcessorBase : public AzureStorageProcessorBase { EXTENSIONAPI static constexpr auto ContainerName = core::PropertyDefinitionBuilder<>::createProperty("Container Name") .withDescription("Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .isRequired(true) .build(); EXTENSIONAPI static constexpr auto StorageAccountName = core::PropertyDefinitionBuilder<>::createProperty("Storage Account Name") @@ -70,7 +71,7 @@ class AzureBlobStorageProcessorBase : public AzureStorageProcessorBase { EXTENSIONAPI static constexpr auto UseManagedIdentityCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Managed Identity Credentials") .withDescription("If true Managed Identity credentials will be used together with the Storage Account Name for authentication.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(AzureStorageProcessorBase::Properties, std::to_array({ diff --git a/extensions/azure/processors/AzureBlobStorageSingleBlobProcessorBase.cpp b/extensions/azure/processors/AzureBlobStorageSingleBlobProcessorBase.cpp index 6414ef7a35..2f2351c1c8 100644 --- a/extensions/azure/processors/AzureBlobStorageSingleBlobProcessorBase.cpp +++ b/extensions/azure/processors/AzureBlobStorageSingleBlobProcessorBase.cpp @@ -32,7 +32,7 @@ bool AzureBlobStorageSingleBlobProcessorBase::setBlobOperationParameters( return false; } - context.getProperty(Blob, params.blob_name, &flow_file); + params.blob_name = context.getProperty(Blob, &flow_file).value_or(""); if (params.blob_name.empty() && (!flow_file.getAttribute("filename", params.blob_name) || params.blob_name.empty())) { logger_->log_error("Blob is not set and default 'filename' attribute could not be found!"); return false; diff --git a/extensions/azure/processors/AzureDataLakeStorageFileProcessorBase.cpp b/extensions/azure/processors/AzureDataLakeStorageFileProcessorBase.cpp index 4defc449a5..f532432bb2 100644 --- a/extensions/azure/processors/AzureDataLakeStorageFileProcessorBase.cpp +++ b/extensions/azure/processors/AzureDataLakeStorageFileProcessorBase.cpp @@ -31,7 +31,7 @@ bool AzureDataLakeStorageFileProcessorBase::setFileOperationCommonParameters( return false; } - context.getProperty(FileName, params.filename, &flow_file); + params.filename = context.getProperty(FileName, &flow_file).value_or(""); if (params.filename.empty() && (!flow_file.getAttribute("filename", params.filename) || params.filename.empty())) { logger_->log_error("No File Name is set and default object key 'filename' attribute could not be found!"); return false; diff --git a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp index 4b2a434f7c..fde8462202 100644 --- a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp @@ -39,16 +39,18 @@ void AzureDataLakeStorageProcessorBase::onSchedule(core::ProcessContext& context credentials_ = *credentials; } -bool AzureDataLakeStorageProcessorBase::setCommonParameters( - storage::AzureDataLakeStorageParameters& params, core::ProcessContext& context, const core::FlowFile* const flow_file) { +bool AzureDataLakeStorageProcessorBase::setCommonParameters(storage::AzureDataLakeStorageParameters& params, + core::ProcessContext& context, + const core::FlowFile* const flow_file) { params.credentials = credentials_; - if (!context.getProperty(FilesystemName, params.file_system_name, flow_file) || params.file_system_name.empty()) { + params.file_system_name = context.getProperty(FilesystemName, flow_file).value_or(""); + if (params.file_system_name.empty()) { logger_->log_error("Filesystem Name '{}' is invalid or empty!", params.file_system_name); return false; } - context.getProperty(DirectoryName, params.directory_name, flow_file); + params.directory_name = context.getProperty(DirectoryName, flow_file).value_or(""); return true; } diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp index a936970199..9844a390ef 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp @@ -30,8 +30,8 @@ namespace org::apache::nifi::minifi::azure::processors { std::tuple> AzureStorageProcessorBase::getCredentialsFromControllerService( core::ProcessContext &context) const { - std::string service_name; - if (!context.getProperty(AzureStorageCredentialsService, service_name) || service_name.empty()) { + std::string service_name = context.getProperty(AzureStorageCredentialsService).value_or(""); + if (service_name.empty()) { return std::make_tuple(GetCredentialsFromControllerResult::CONTROLLER_NAME_EMPTY, std::nullopt); } diff --git a/extensions/azure/processors/FetchAzureBlobStorage.cpp b/extensions/azure/processors/FetchAzureBlobStorage.cpp index 427a500795..fa0a065f22 100644 --- a/extensions/azure/processors/FetchAzureBlobStorage.cpp +++ b/extensions/azure/processors/FetchAzureBlobStorage.cpp @@ -40,13 +40,13 @@ std::optional FetchAzureBlobStorage::b } std::string value; - if (context.getProperty(RangeStart, value, &flow_file)) { - params.range_start = std::stoull(value); + if (auto range_start = utils::parseOptionalU64Property(context, RangeStart, &flow_file)) { + params.range_start = *range_start; logger_->log_debug("Range Start property set to {}", *params.range_start); } - if (context.getProperty(RangeLength, value, &flow_file)) { - params.range_length = std::stoull(value); + if (auto range_length = utils::parseOptionalU64Property(context, RangeLength, &flow_file)) { + params.range_length = *range_length; logger_->log_debug("Range Length property set to {}", *params.range_length); } diff --git a/extensions/azure/processors/FetchAzureDataLakeStorage.cpp b/extensions/azure/processors/FetchAzureDataLakeStorage.cpp index b8a758d747..4dffd0392a 100644 --- a/extensions/azure/processors/FetchAzureDataLakeStorage.cpp +++ b/extensions/azure/processors/FetchAzureDataLakeStorage.cpp @@ -23,6 +23,7 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::azure::processors { @@ -38,19 +39,18 @@ std::optional FetchAzureDataLakeSt return std::nullopt; } - std::string value; - if (context.getProperty(RangeStart, value, &flow_file)) { - params.range_start = std::stoull(value); + if (auto range_start = utils::parseOptionalU64Property(context, RangeStart, &flow_file)) { + params.range_start = *range_start; logger_->log_debug("Range Start property set to {}", *params.range_start); } - if (context.getProperty(RangeLength, value, &flow_file)) { - params.range_length = std::stoull(value); + if (auto range_length = utils::parseOptionalU64Property(context, RangeLength, &flow_file)) { + params.range_length = *range_length; logger_->log_debug("Range Length property set to {}", *params.range_length); } - if (context.getProperty(NumberOfRetries, value, &flow_file)) { - params.number_of_retries = std::stoull(value); + if (auto number_of_retries = utils::parseOptionalU64Property(context, NumberOfRetries, &flow_file)) { + params.number_of_retries = *number_of_retries; logger_->log_debug("Number Of Retries property set to {}", *params.number_of_retries); } diff --git a/extensions/azure/processors/FetchAzureDataLakeStorage.h b/extensions/azure/processors/FetchAzureDataLakeStorage.h index 0002cfb42b..ed90841b1e 100644 --- a/extensions/azure/processors/FetchAzureDataLakeStorage.h +++ b/extensions/azure/processors/FetchAzureDataLakeStorage.h @@ -47,7 +47,7 @@ class FetchAzureDataLakeStorage final : public AzureDataLakeStorageFileProcessor .build(); EXTENSIONAPI static constexpr auto NumberOfRetries = core::PropertyDefinitionBuilder<>::createProperty("Number of Retries") .withDescription("The number of automatic retries to perform if the download fails.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("0") .supportsExpressionLanguage(true) .build(); diff --git a/extensions/azure/processors/ListAzureBlobStorage.cpp b/extensions/azure/processors/ListAzureBlobStorage.cpp index 35fe0fee22..e34274ecb8 100644 --- a/extensions/azure/processors/ListAzureBlobStorage.cpp +++ b/extensions/azure/processors/ListAzureBlobStorage.cpp @@ -56,7 +56,7 @@ std::optional ListAzureBlobStorage::bui return std::nullopt; } - context.getProperty(Prefix, params.prefix, nullptr); + params.prefix = context.getProperty(Prefix).value_or(""); return params; } diff --git a/extensions/azure/processors/ListAzureDataLakeStorage.cpp b/extensions/azure/processors/ListAzureDataLakeStorage.cpp index 4311f7d1db..f9fa48dd7e 100644 --- a/extensions/azure/processors/ListAzureDataLakeStorage.cpp +++ b/extensions/azure/processors/ListAzureDataLakeStorage.cpp @@ -69,15 +69,11 @@ std::optional ListAzureDataLakeStor return std::nullopt; } - if (!context.getProperty(RecurseSubdirectories, params.recurse_subdirectories)) { - logger_->log_error("Recurse Subdirectories property missing or invalid"); - return std::nullopt; - } + params.recurse_subdirectories = utils::parseBoolProperty(context, RecurseSubdirectories); auto createFilterRegex = [&context](std::string_view property_name) -> std::optional { try { - std::string filter_str; - context.getProperty(property_name, filter_str); + std::string filter_str = context.getProperty(property_name).value_or(""); if (!filter_str.empty()) { return minifi::utils::Regex(filter_str); } diff --git a/extensions/azure/processors/ListAzureDataLakeStorage.h b/extensions/azure/processors/ListAzureDataLakeStorage.h index 67d194b508..47467373ce 100644 --- a/extensions/azure/processors/ListAzureDataLakeStorage.h +++ b/extensions/azure/processors/ListAzureDataLakeStorage.h @@ -41,7 +41,7 @@ class ListAzureDataLakeStorage final : public AzureDataLakeStorageProcessorBase EXTENSIONAPI static constexpr auto RecurseSubdirectories = core::PropertyDefinitionBuilder<>::createProperty("Recurse Subdirectories") .withDescription("Indicates whether to list files from subdirectories of the directory") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("true") .build(); EXTENSIONAPI static constexpr auto FileFilter = core::PropertyDefinitionBuilder<>::createProperty("File Filter") diff --git a/extensions/azure/processors/PutAzureBlobStorage.cpp b/extensions/azure/processors/PutAzureBlobStorage.cpp index c917efe9fb..fd3576641a 100644 --- a/extensions/azure/processors/PutAzureBlobStorage.cpp +++ b/extensions/azure/processors/PutAzureBlobStorage.cpp @@ -23,6 +23,8 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" +#include "utils/ProcessorConfigUtils.h" + namespace org::apache::nifi::minifi::azure::processors { @@ -34,7 +36,7 @@ void PutAzureBlobStorage::initialize() { void PutAzureBlobStorage::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { AzureBlobStorageProcessorBase::onSchedule(context, session_factory); - context.getProperty(CreateContainer, create_container_); + create_container_ = minifi::utils::parseBoolProperty(context, CreateContainer); } std::optional PutAzureBlobStorage::buildPutAzureBlobStorageParameters( diff --git a/extensions/azure/processors/PutAzureBlobStorage.h b/extensions/azure/processors/PutAzureBlobStorage.h index fb2ad6765d..31b18ed4db 100644 --- a/extensions/azure/processors/PutAzureBlobStorage.h +++ b/extensions/azure/processors/PutAzureBlobStorage.h @@ -49,7 +49,7 @@ class PutAzureBlobStorage final : public AzureBlobStorageSingleBlobProcessorBase "Permission to list containers is required. If false, this check is not made, but the Put operation will " "fail if the container does not exist.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(AzureBlobStorageSingleBlobProcessorBase::Properties, std::to_array({CreateContainer})); diff --git a/extensions/bustache/ApplyTemplate.cpp b/extensions/bustache/ApplyTemplate.cpp index 0e26757e0e..6764aa8425 100644 --- a/extensions/bustache/ApplyTemplate.cpp +++ b/extensions/bustache/ApplyTemplate.cpp @@ -19,14 +19,15 @@ */ #include "ApplyTemplate.h" -#include #include #include +#include #include -#include "core/Resource.h" #include "bustache/model.hpp" #include "bustache/render/string.hpp" +#include "core/Resource.h" +#include "core/ProcessContext.h" namespace org::apache::nifi::minifi::processors { @@ -41,8 +42,7 @@ void ApplyTemplate::onTrigger(core::ProcessContext& context, core::ProcessSessio return; } - std::string template_file; - context.getProperty(Template, template_file, flow_file.get()); + std::string template_file = context.getProperty(Template.name, flow_file.get()).value_or(""); session.write(flow_file, [&template_file, &flow_file, this](const auto& output_stream) { logger_->log_info("ApplyTemplate reading template file from {}", template_file); // TODO(szaszm): we might want to return to memory-mapped input files when the next todo is done. Until then, the agents stores the whole result in memory anyway, so no point in not doing the same diff --git a/extensions/civetweb/processors/ListenHTTP.cpp b/extensions/civetweb/processors/ListenHTTP.cpp index 1e12f72b16..39cf718fac 100644 --- a/extensions/civetweb/processors/ListenHTTP.cpp +++ b/extensions/civetweb/processors/ListenHTTP.cpp @@ -28,6 +28,7 @@ #include "core/Resource.h" #include "utils/gsl.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::processors { @@ -39,71 +40,26 @@ void ListenHTTP::initialize() { } void ListenHTTP::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - std::string basePath; + std::string base_path = context.getProperty(BasePath) | utils::expect("ListenHTTP::BasePath has default value"); - if (!context.getProperty(BasePath, basePath)) { - static_assert(BasePath.default_value); - logger_->log_info("{} attribute is missing, so default value of {} will be used", BasePath.name, *BasePath.default_value); - basePath = *BasePath.default_value; - } - - basePath.insert(0, "/"); + base_path.insert(0, "/"); - if (!context.getProperty(Port, listeningPort)) { - logger_->log_error("{} attribute is missing or invalid", Port.name); - return; - } + listeningPort = utils::parseProperty(context, Port); bool randomPort = listeningPort == "0"; - std::string authDNPattern; - if (context.getProperty(AuthorizedDNPattern, authDNPattern) && !authDNPattern.empty()) { - logger_->log_debug("ListenHTTP using {}: {}", AuthorizedDNPattern.name, authDNPattern); - } else { - authDNPattern = ".*"; - logger_->log_debug("Authorized DN Pattern not set or invalid, using default '{}' pattern", authDNPattern); - } + auto authDNPattern = context.getProperty(AuthorizedDNPattern).value_or(".*"); + const auto sslCertFile = context.getProperty(SSLCertificate).value_or(""); + const auto sslCertAuthorityFile = context.getProperty(SSLCertificateAuthority).value_or(""); + const auto sslVerifyPeer = context.getProperty(SSLVerifyPeer).value_or(""); + const auto sslMinVer = context.getProperty(SSLMinimumVersion).value_or(""); - std::string sslCertFile; - - if (context.getProperty(SSLCertificate, sslCertFile) && !sslCertFile.empty()) { - logger_->log_debug("ListenHTTP using {}: {}", SSLCertificate.name, sslCertFile); - } - - // Read further TLS/SSL options only if TLS/SSL usage is implied by virtue of certificate value being set - std::string sslCertAuthorityFile; - std::string sslVerifyPeer; - std::string sslMinVer; - - if (!sslCertFile.empty()) { - if (context.getProperty(SSLCertificateAuthority, sslCertAuthorityFile) && !sslCertAuthorityFile.empty()) { - logger_->log_debug("ListenHTTP using {}: {}", SSLCertificateAuthority.name, sslCertAuthorityFile); - } - - if (context.getProperty(SSLVerifyPeer, sslVerifyPeer)) { - if (sslVerifyPeer.empty() || sslVerifyPeer == "no") { - logger_->log_debug("ListenHTTP will not verify peers"); - } else { - logger_->log_debug("ListenHTTP will verify peers"); - } - } else { - logger_->log_debug("ListenHTTP will not verify peers"); - } - - if (context.getProperty(SSLMinimumVersion, sslMinVer)) { - logger_->log_debug("ListenHTTP using {}: {}", SSLMinimumVersion.name, sslMinVer); - } - } - - std::string headersAsAttributesPattern; - - if (context.getProperty(HeadersAsAttributesRegex, headersAsAttributesPattern) && !headersAsAttributesPattern.empty()) { - logger_->log_debug("ListenHTTP using {}: {}", HeadersAsAttributesRegex.name, headersAsAttributesPattern); - } + std::string headersAsAttributesPattern = context.getProperty(HeadersAsAttributesRegex).value_or(""); + logger_->log_debug("ListenHTTP using {}: {}", HeadersAsAttributesRegex.name, headersAsAttributesPattern); auto numThreads = getMaxConcurrentTasks(); - logger_->log_info("ListenHTTP starting HTTP server on port {} and path {} with {} threads", randomPort ? "random" : listeningPort, basePath, numThreads); + logger_->log_info("ListenHTTP starting HTTP server on port {} and path {} with {} threads", randomPort ? "random" : listeningPort, base_path, numThreads); // Initialize web server std::vector options; @@ -148,17 +104,18 @@ void ListenHTTP::onSchedule(core::ProcessContext& context, core::ProcessSessionF server_ = std::make_unique(options, &callbacks_, &logger_); - context.getProperty(BatchSize, batch_size_); + batch_size_ = utils::parseU64Property(context, BatchSize); logger_->log_debug("ListenHTTP using {}: {}", BatchSize.name, batch_size_); std::optional flow_id; - if (auto flow_version = context.getProcessorNode()->getFlowIdentifier()) { + if (auto flow_version = context.getProcessor().getFlowIdentifier()) { flow_id = flow_version->getFlowId(); } - handler_ = std::make_unique(basePath, flow_id, context.getProperty(BufferSize).value_or(0), std::move(authDNPattern), + const auto buffer_size = utils::parseU64Property(context, BufferSize); + handler_ = std::make_unique(base_path, flow_id, buffer_size, std::move(authDNPattern), headersAsAttributesPattern.empty() ? std::nullopt : std::make_optional(headersAsAttributesPattern)); - server_->addHandler(basePath, handler_.get()); + server_->addHandler(base_path, handler_.get()); if (randomPort) { const auto& vec = server_->getListeningPorts(); diff --git a/extensions/civetweb/processors/ListenHTTP.h b/extensions/civetweb/processors/ListenHTTP.h index 4a0763df7c..bb89387430 100644 --- a/extensions/civetweb/processors/ListenHTTP.h +++ b/extensions/civetweb/processors/ListenHTTP.h @@ -78,7 +78,7 @@ class ListenHTTP : public core::ProcessorImpl { EXTENSIONAPI static constexpr auto Port = core::PropertyDefinitionBuilder<>::createProperty("Listening Port") .withDescription("The Port to listen on for incoming connections. 0 means port is going to be selected randomly.") .isRequired(true) - .withPropertyType(core::StandardPropertyTypes::LISTEN_PORT_TYPE) + .withValidator(core::StandardPropertyTypes::PORT_VALIDATOR) .withDefaultValue("80") .build(); EXTENSIONAPI static constexpr auto AuthorizedDNPattern = core::PropertyDefinitionBuilder<>::createProperty("Authorized DN Pattern") @@ -109,13 +109,13 @@ class ListenHTTP : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto BatchSize = core::PropertyDefinitionBuilder<>::createProperty("Batch Size") .withDescription("Maximum number of buffered requests to be processed in a single batch. If set to zero all buffered requests are processed.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue(ListenHTTP::DEFAULT_BUFFER_SIZE_STR) .build(); EXTENSIONAPI static constexpr auto BufferSize = core::PropertyDefinitionBuilder<>::createProperty("Buffer Size") .withDescription("Maximum number of HTTP Requests allowed to be buffered before processing them when the processor is triggered. " "If the buffer full, the request is refused. If set to zero the buffer is unlimited.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue(ListenHTTP::DEFAULT_BUFFER_SIZE_STR) .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ diff --git a/extensions/civetweb/tests/ListenHTTPTests.cpp b/extensions/civetweb/tests/ListenHTTPTests.cpp index f67a3f89fb..a21add5bd2 100644 --- a/extensions/civetweb/tests/ListenHTTPTests.cpp +++ b/extensions/civetweb/tests/ListenHTTPTests.cpp @@ -157,10 +157,7 @@ class ListenHTTPTestsFixture { void check_content_type(minifi::http::HTTPClient& client) { if (endpoint == "test") { - std::string content_type; - if (!update_attribute->getDynamicProperty("mime.type", content_type)) { - content_type = "application/octet-stream"; - } + const auto content_type = update_attribute->getDynamicProperty("mime.type").value_or("application/octet-stream"); REQUIRE(content_type == minifi::utils::string::trim(client.getResponseHeaderMap().at("Content-type"))); REQUIRE("19" == minifi::utils::string::trim(client.getResponseHeaderMap().at("Content-length"))); } else { @@ -206,7 +203,7 @@ class ListenHTTPTestsFixture { if (client_to_use) { REQUIRE(response_expectations.size() == 1); } - auto* proc = dynamic_cast(plan->getCurrentContext()->getProcessorNode()->getProcessor()); + auto* proc = dynamic_cast(&plan->getCurrentContext()->getProcessor()); REQUIRE(proc); std::vector client_threads; @@ -702,7 +699,7 @@ TEST_CASE("ListenHTTP bored yield", "[listenhttp][bored][yield]") { using processors::ListenHTTP; SingleProcessorTestController controller{std::make_unique("listenhttp")}; auto listen_http = controller.getProcessor(); - listen_http->setProperty(ListenHTTP::Port, "0"); + listen_http->setProperty(ListenHTTP::Port.name, "0"); REQUIRE(!listen_http->isYield()); const auto output = controller.trigger(); diff --git a/extensions/couchbase/controllerservices/CouchbaseClusterService.cpp b/extensions/couchbase/controllerservices/CouchbaseClusterService.cpp index ac340f8640..b097226aa8 100644 --- a/extensions/couchbase/controllerservices/CouchbaseClusterService.cpp +++ b/extensions/couchbase/controllerservices/CouchbaseClusterService.cpp @@ -216,12 +216,10 @@ void CouchbaseClusterService::initialize() { } void CouchbaseClusterService::onEnable() { - std::string connection_string; - getProperty(ConnectionString, connection_string); - std::string username; - getProperty(UserName, username); - std::string password; - getProperty(UserPassword, password); + std::string connection_string = getProperty(ConnectionString.name) | utils::expect("required property"); + std::string username = getProperty(UserName.name).value_or(""); + std::string password = getProperty(UserPassword.name).value_or(""); + if (connection_string.empty()) { throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing connection string"); } @@ -252,7 +250,7 @@ void CouchbaseClusterService::onEnable() { gsl::not_null> CouchbaseClusterService::getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property) { std::shared_ptr couchbase_cluster_service; if (auto connection_controller_name = context.getProperty(property)) { - couchbase_cluster_service = std::dynamic_pointer_cast(context.getControllerService(*connection_controller_name, context.getProcessorNode()->getUUID())); + couchbase_cluster_service = std::dynamic_pointer_cast(context.getControllerService(*connection_controller_name, context.getProcessor().getUUID())); } if (!couchbase_cluster_service) { throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing Couchbase Cluster Service"); diff --git a/extensions/couchbase/processors/GetCouchbaseKey.cpp b/extensions/couchbase/processors/GetCouchbaseKey.cpp index 89cf160f64..02fd95e973 100644 --- a/extensions/couchbase/processors/GetCouchbaseKey.cpp +++ b/extensions/couchbase/processors/GetCouchbaseKey.cpp @@ -38,25 +38,20 @@ void GetCouchbaseKey::onTrigger(core::ProcessContext& context, core::ProcessSess } CouchbaseCollection collection; - if (!context.getProperty(BucketName, collection.bucket_name, flow_file.get()) || collection.bucket_name.empty()) { + if (const auto bucket_name = utils::parseOptionalProperty(context, BucketName, flow_file.get())) { + collection.bucket_name = *bucket_name; + } else { logger_->log_error("Bucket '{}' is invalid or empty!", collection.bucket_name); session.transfer(flow_file, Failure); return; } - if (!context.getProperty(ScopeName, collection.scope_name, flow_file.get()) || collection.scope_name.empty()) { - collection.scope_name = ::couchbase::scope::default_name; - } - - if (!context.getProperty(CollectionName, collection.collection_name, flow_file.get()) || collection.collection_name.empty()) { - collection.collection_name = ::couchbase::collection::default_name; - } - - std::string document_id; - if (!context.getProperty(DocumentId, document_id, flow_file.get()) || document_id.empty()) { - auto ff_content = session.readBuffer(flow_file).buffer; - document_id = std::string(reinterpret_cast(ff_content.data()), ff_content.size()); - } + collection.scope_name = utils::parseOptionalProperty(context, ScopeName, flow_file.get()).value_or(::couchbase::scope::default_name); + collection.collection_name = utils::parseOptionalProperty(context, CollectionName, flow_file.get()).value_or(::couchbase::collection::default_name); + std::string document_id = utils::parseOptionalProperty(context, DocumentId, flow_file.get()) | utils::valueOrElse([&flow_file, &session] { + const auto ff_content = session.readBuffer(flow_file).buffer; + return std::string(reinterpret_cast(ff_content.data()), ff_content.size()); + }); if (document_id.empty()) { logger_->log_error("Document ID is empty, transferring FlowFile to failure relationship"); @@ -64,8 +59,7 @@ void GetCouchbaseKey::onTrigger(core::ProcessContext& context, core::ProcessSess return; } - std::string attribute_to_put_result_to; - context.getProperty(PutValueToAttribute, attribute_to_put_result_to, flow_file.get()); + std::string attribute_to_put_result_to = utils::parseOptionalProperty(context, PutValueToAttribute, flow_file.get()).value_or(""); if (auto get_result = couchbase_cluster_service_->get(collection, document_id, document_type_)) { if (!attribute_to_put_result_to.empty()) { diff --git a/extensions/couchbase/processors/GetCouchbaseKey.h b/extensions/couchbase/processors/GetCouchbaseKey.h index bec91841cd..e89de0df5c 100644 --- a/extensions/couchbase/processors/GetCouchbaseKey.h +++ b/extensions/couchbase/processors/GetCouchbaseKey.h @@ -45,15 +45,18 @@ class GetCouchbaseKey final : public core::AbstractProcessor { .withDescription("The name of bucket to access.") .withDefaultValue("default") .isRequired(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto ScopeName = core::PropertyDefinitionBuilder<>::createProperty("Scope Name") .withDescription("Scope to use inside the bucket. If not specified, the _default scope is used.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto CollectionName = core::PropertyDefinitionBuilder<>::createProperty("Collection Name") .withDescription("Collection to use inside the bucket scope. If not specified, the _default collection is used.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto DocumentType = core::PropertyDefinitionBuilder<3>::createProperty("Document Type") .withDescription("Content type of the retrieved value.") @@ -64,6 +67,7 @@ class GetCouchbaseKey final : public core::AbstractProcessor { EXTENSIONAPI static constexpr auto DocumentId = core::PropertyDefinitionBuilder<>::createProperty("Document Id") .withDescription("A static, fixed Couchbase document id, or an expression to construct the Couchbase document id.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto PutValueToAttribute = core::PropertyDefinitionBuilder<>::createProperty("Put Value to Attribute") .withDescription("If set, the retrieved value will be put into an attribute of the FlowFile instead of a the content of the FlowFile. " diff --git a/extensions/couchbase/processors/PutCouchbaseKey.cpp b/extensions/couchbase/processors/PutCouchbaseKey.cpp index 6f8ed54ceb..7c7edd4a02 100644 --- a/extensions/couchbase/processors/PutCouchbaseKey.cpp +++ b/extensions/couchbase/processors/PutCouchbaseKey.cpp @@ -40,24 +40,19 @@ void PutCouchbaseKey::onTrigger(core::ProcessContext& context, core::ProcessSess } CouchbaseCollection collection; - if (!context.getProperty(BucketName, collection.bucket_name, flow_file.get()) || collection.bucket_name.empty()) { + if (const auto bucket_name = utils::parseOptionalProperty(context, BucketName, flow_file.get())) { + collection.bucket_name = *bucket_name; + } else { logger_->log_error("Bucket '{}' is invalid or empty!", collection.bucket_name); session.transfer(flow_file, Failure); return; } - if (!context.getProperty(ScopeName, collection.scope_name, flow_file.get()) || collection.scope_name.empty()) { - collection.scope_name = ::couchbase::scope::default_name; - } - - if (!context.getProperty(CollectionName, collection.collection_name, flow_file.get()) || collection.collection_name.empty()) { - collection.collection_name = ::couchbase::collection::default_name; - } - - std::string document_id; - if (!context.getProperty(DocumentId, document_id, flow_file.get()) || document_id.empty()) { - document_id = flow_file->getAttribute(core::SpecialFlowAttribute::UUID).value_or(utils::IdGenerator::getIdGenerator()->generate().to_string()); - } + collection.scope_name = utils::parseOptionalProperty(context, ScopeName, flow_file.get()).value_or(::couchbase::scope::default_name); + collection.collection_name = utils::parseOptionalProperty(context, CollectionName, flow_file.get()).value_or(::couchbase::collection::default_name); + std::string document_id = utils::parseOptionalProperty(context, DocumentId, flow_file.get()) + | utils::orElse([&flow_file] { return flow_file->getAttribute(core::SpecialFlowAttribute::UUID); }) + | utils::valueOrElse([] { return utils::IdGenerator::getIdGenerator()->generate().to_string(); }); ::couchbase::upsert_options options; options.durability(persist_to_, replicate_to_); diff --git a/extensions/couchbase/processors/PutCouchbaseKey.h b/extensions/couchbase/processors/PutCouchbaseKey.h index a1e3601af3..5fef374328 100644 --- a/extensions/couchbase/processors/PutCouchbaseKey.h +++ b/extensions/couchbase/processors/PutCouchbaseKey.h @@ -82,16 +82,19 @@ class PutCouchbaseKey final : public core::AbstractProcessor { EXTENSIONAPI static constexpr auto BucketName = core::PropertyDefinitionBuilder<>::createProperty("Bucket Name") .withDescription("The name of bucket to access.") .withDefaultValue("default") + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .isRequired(true) .supportsExpressionLanguage(true) .build(); EXTENSIONAPI static constexpr auto ScopeName = core::PropertyDefinitionBuilder<>::createProperty("Scope Name") .withDescription("Scope to use inside the bucket. If not specified, the _default scope is used.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto CollectionName = core::PropertyDefinitionBuilder<>::createProperty("Collection Name") .withDescription("Collection to use inside the bucket scope. If not specified, the _default collection is used.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto DocumentType = core::PropertyDefinitionBuilder()>::createProperty("Document Type") .withDescription("Content type to store data as.") @@ -103,6 +106,7 @@ class PutCouchbaseKey final : public core::AbstractProcessor { .withDescription("A static, fixed Couchbase document id, or an expression to construct the Couchbase document id. " "If not specified, either the FlowFile uuid attribute or if that's not found a generated uuid will be used.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto PersistTo = core::PropertyDefinitionBuilder<6>::createProperty("Persist To") .withDescription("Durability constraint about disk persistence.") diff --git a/extensions/couchbase/tests/GetCouchbaseKeyTests.cpp b/extensions/couchbase/tests/GetCouchbaseKeyTests.cpp index 185d99ac01..99f6090bdf 100644 --- a/extensions/couchbase/tests/GetCouchbaseKeyTests.cpp +++ b/extensions/couchbase/tests/GetCouchbaseKeyTests.cpp @@ -48,7 +48,7 @@ class GetCouchbaseKeyTestController : public TestController { auto controller_service_node = controller_.plan->addController("MockCouchbaseClusterService", "MockCouchbaseClusterService"); mock_couchbase_cluster_service_ = std::dynamic_pointer_cast(controller_service_node->getControllerServiceImplementation()); gsl_Assert(mock_couchbase_cluster_service_); - proc_->setProperty(processors::GetCouchbaseKey::CouchbaseClusterControllerService, "MockCouchbaseClusterService"); + proc_->setProperty(processors::GetCouchbaseKey::CouchbaseClusterControllerService.name, "MockCouchbaseClusterService"); } void verifyResults(const minifi::test::ProcessorTriggerResult& results, const minifi::core::Relationship& expected_result, const ExpectedCallOptions& expected_call_options, @@ -89,8 +89,7 @@ class GetCouchbaseKeyTestController : public TestController { CHECK(flow_file->getAttribute("couchbase.doc.id").value() == expected_call_options.doc_id); CHECK(flow_file->getAttribute("couchbase.doc.cas").value() == std::to_string(COUCHBASE_GET_RESULT_CAS)); CHECK(flow_file->getAttribute("couchbase.doc.expiry").value() == COUCHBASE_GET_RESULT_EXPIRY); - std::string value; - proc_->getProperty(processors::GetCouchbaseKey::PutValueToAttribute, value); + std::string value = proc_->getProperty(processors::GetCouchbaseKey::PutValueToAttribute.name).value_or(""); if (!value.empty()) { CHECK(flow_file->getAttribute(value).value() == COUCHBASE_GET_RESULT_CONTENT); CHECK(controller_.plan->getContent(flow_file) == input); @@ -106,19 +105,23 @@ class GetCouchbaseKeyTestController : public TestController { }; TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Invalid Couchbase cluster controller service", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::CouchbaseClusterControllerService, "invalid"); + proc_->setProperty(processors::GetCouchbaseKey::CouchbaseClusterControllerService.name, "invalid"); REQUIRE_THROWS_AS(controller_.trigger({minifi::test::InputFlowFileData{"couchbase_id"}}), minifi::Exception); } -TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Invalid bucket name", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, ""); +TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Can't set empty bucket name", "[getcouchbasekey]") { + CHECK_FALSE(proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "")); +} + +TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Empty evaluated bucket name", "[getcouchbasekey]") { + CHECK(proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "${missing_attr}")); auto results = controller_.trigger({minifi::test::InputFlowFileData{"couchbase_id"}}); REQUIRE(results[processors::GetCouchbaseKey::Failure].size() == 1); REQUIRE(LogTestController::getInstance().contains("Bucket '' is invalid or empty!", 1s)); } TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Document ID is empty and no content is present to use", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); auto results = controller_.trigger({minifi::test::InputFlowFileData{""}}); REQUIRE(results.at(processors::GetCouchbaseKey::Success).empty()); REQUIRE(results.at(processors::GetCouchbaseKey::Failure).size() == 1); @@ -126,25 +129,33 @@ TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Document ID is empty and no con REQUIRE(LogTestController::getInstance().contains("Document ID is empty, transferring FlowFile to failure relationship", 1s)); } +TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Document ID evaluates to be empty", "[getcouchbasekey]") { + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); + CHECK(proc_->setProperty(processors::GetCouchbaseKey::DocumentId.name, "${missing_attr}")); + const std::string input = "couchbase_id"; + auto results = controller_.trigger({minifi::test::InputFlowFileData{input}}); + verifyResults(results, processors::GetCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "_default", "_default", input, couchbase::CouchbaseValueType::Json}, input); +} + TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Get succeeeds with default properties", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); const std::string input = "couchbase_id"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input}}); verifyResults(results, processors::GetCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "_default", "_default", input, couchbase::CouchbaseValueType::Json}, input); } TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Get succeeeds with optional properties", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); - proc_->setProperty(processors::GetCouchbaseKey::ScopeName, "scope1"); - proc_->setProperty(processors::GetCouchbaseKey::CollectionName, "collection1"); - proc_->setProperty(processors::GetCouchbaseKey::DocumentId, "important_doc"); - proc_->setProperty(processors::GetCouchbaseKey::DocumentType, "Binary"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::ScopeName.name, "scope1"); + proc_->setProperty(processors::GetCouchbaseKey::CollectionName.name, "collection1"); + proc_->setProperty(processors::GetCouchbaseKey::DocumentId.name, "important_doc"); + proc_->setProperty(processors::GetCouchbaseKey::DocumentType.name, "Binary"); auto results = controller_.trigger({minifi::test::InputFlowFileData{""}}); verifyResults(results, processors::GetCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "scope1", "collection1", "important_doc", couchbase::CouchbaseValueType::Binary}, ""); } TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Get fails with default properties", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); mock_couchbase_cluster_service_->setGetError(CouchbaseErrorType::FATAL); const std::string input = "couchbase_id"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input}}); @@ -152,7 +163,7 @@ TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Get fails with default properti } TEST_CASE_METHOD(GetCouchbaseKeyTestController, "FlowFile is transferred to retry relationship when temporary error is returned", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); mock_couchbase_cluster_service_->setGetError(CouchbaseErrorType::TEMPORARY); const std::string input = "couchbase_id"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input}}); @@ -160,9 +171,9 @@ TEST_CASE_METHOD(GetCouchbaseKeyTestController, "FlowFile is transferred to retr } TEST_CASE_METHOD(GetCouchbaseKeyTestController, "Get result is written to attribute", "[getcouchbasekey]") { - proc_->setProperty(processors::GetCouchbaseKey::BucketName, "mybucket"); - proc_->setProperty(processors::GetCouchbaseKey::DocumentType, "String"); - proc_->setProperty(processors::GetCouchbaseKey::PutValueToAttribute, "myattribute"); + proc_->setProperty(processors::GetCouchbaseKey::BucketName.name, "mybucket"); + proc_->setProperty(processors::GetCouchbaseKey::DocumentType.name, "String"); + proc_->setProperty(processors::GetCouchbaseKey::PutValueToAttribute.name, "myattribute"); const std::string input = "couchbase_id"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input}}); verifyResults(results, processors::GetCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "_default", "_default", input, couchbase::CouchbaseValueType::String}, input); diff --git a/extensions/couchbase/tests/PutCouchbaseKeyTests.cpp b/extensions/couchbase/tests/PutCouchbaseKeyTests.cpp index 56970289f8..1627eb000b 100644 --- a/extensions/couchbase/tests/PutCouchbaseKeyTests.cpp +++ b/extensions/couchbase/tests/PutCouchbaseKeyTests.cpp @@ -52,7 +52,7 @@ class PutCouchbaseKeyTestController : public TestController { auto controller_service_node = controller_.plan->addController("MockCouchbaseClusterService", "MockCouchbaseClusterService"); mock_couchbase_cluster_service_ = std::dynamic_pointer_cast(controller_service_node->getControllerServiceImplementation()); gsl_Assert(mock_couchbase_cluster_service_); - proc_->setProperty(processors::PutCouchbaseKey::CouchbaseClusterControllerService, "MockCouchbaseClusterService"); + proc_->setProperty(processors::PutCouchbaseKey::CouchbaseClusterControllerService.name, "MockCouchbaseClusterService"); } [[nodiscard]] static std::vector stringToByteVector(const std::string& str) { @@ -119,19 +119,23 @@ class PutCouchbaseKeyTestController : public TestController { }; TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Invalid Couchbase cluster controller service", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::CouchbaseClusterControllerService, "invalid"); + proc_->setProperty(processors::PutCouchbaseKey::CouchbaseClusterControllerService.name, "invalid"); REQUIRE_THROWS_AS(controller_.trigger({minifi::test::InputFlowFileData{"{\"name\": \"John\"}\n{\"name\": \"Jill\"}", {{"uuid", TEST_UUID}}}}), minifi::Exception); } -TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Invalid bucket name", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::BucketName, ""); - auto results = controller_.trigger({minifi::test::InputFlowFileData{"{\"name\": \"John\"}\n{\"name\": \"Jill\"}", {{"uuid", TEST_UUID}}}}); +TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Can't set empty bucket name", "[getcouchbasekey]") { + CHECK_FALSE(proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "")); +} + +TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Empty evaluated bucket name", "[getcouchbasekey]") { + CHECK(proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "${bucket_name}")); + auto results = controller_.trigger({minifi::test::InputFlowFileData{"couchbase_id"}}); REQUIRE(results[processors::PutCouchbaseKey::Failure].size() == 1); REQUIRE(LogTestController::getInstance().contains("Bucket '' is invalid or empty!", 1s)); } TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put succeeeds with default properties", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "mybucket"); const std::string input = "{\"name\": \"John\"}\n{\"name\": \"Jill\"}"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input, {{"uuid", TEST_UUID}}}}); verifyResults(results, processors::PutCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "_default", "_default", @@ -139,13 +143,13 @@ TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put succeeeds with default prop } TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put succeeeds with optional properties", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::BucketName, "mybucket"); - proc_->setProperty(processors::PutCouchbaseKey::ScopeName, "scope1"); - proc_->setProperty(processors::PutCouchbaseKey::CollectionName, "collection1"); - proc_->setProperty(processors::PutCouchbaseKey::DocumentType, "Binary"); - proc_->setProperty(processors::PutCouchbaseKey::DocumentId, "important_doc"); - proc_->setProperty(processors::PutCouchbaseKey::PersistTo, "ACTIVE"); - proc_->setProperty(processors::PutCouchbaseKey::ReplicateTo, "TWO"); + proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "mybucket"); + proc_->setProperty(processors::PutCouchbaseKey::ScopeName.name, "scope1"); + proc_->setProperty(processors::PutCouchbaseKey::CollectionName.name, "collection1"); + proc_->setProperty(processors::PutCouchbaseKey::DocumentType.name, "Binary"); + proc_->setProperty(processors::PutCouchbaseKey::DocumentId.name, "important_doc"); + proc_->setProperty(processors::PutCouchbaseKey::PersistTo.name, "ACTIVE"); + proc_->setProperty(processors::PutCouchbaseKey::ReplicateTo.name, "TWO"); const std::string input = "{\"name\": \"John\"}\n{\"name\": \"Jill\"}"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input, {{"uuid", TEST_UUID}}}}); verifyResults(results, processors::PutCouchbaseKey::Success, ExpectedCallOptions{"mybucket", "scope1", "collection1", ::couchbase::persist_to::active, @@ -153,7 +157,7 @@ TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put succeeeds with optional pro } TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put fails with default properties", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "mybucket"); mock_couchbase_cluster_service_->setUpsertError(CouchbaseErrorType::FATAL); const std::string input = "{\"name\": \"John\"}\n{\"name\": \"Jill\"}"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input, {{"uuid", TEST_UUID}}}}); @@ -162,7 +166,7 @@ TEST_CASE_METHOD(PutCouchbaseKeyTestController, "Put fails with default properti } TEST_CASE_METHOD(PutCouchbaseKeyTestController, "FlowFile is transferred to retry relationship when temporary error is returned", "[putcouchbasekey]") { - proc_->setProperty(processors::PutCouchbaseKey::BucketName, "mybucket"); + proc_->setProperty(processors::PutCouchbaseKey::BucketName.name, "mybucket"); mock_couchbase_cluster_service_->setUpsertError(CouchbaseErrorType::TEMPORARY); const std::string input = "{\"name\": \"John\"}\n{\"name\": \"Jill\"}"; auto results = controller_.trigger({minifi::test::InputFlowFileData{input, {{"uuid", TEST_UUID}}}}); diff --git a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp index 62c2a96131..b09bbd4f38 100644 --- a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp +++ b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp @@ -15,10 +15,13 @@ * limitations under the License. */ +#include "ElasticsearchCredentialsControllerService.h" + + #include -#include "ElasticsearchCredentialsControllerService.h" #include "core/Resource.h" +#include "minifi-cpp/Exception.h" namespace org::apache::nifi::minifi::extensions::elasticsearch { @@ -27,11 +30,10 @@ void ElasticsearchCredentialsControllerService::initialize() { } void ElasticsearchCredentialsControllerService::onEnable() { - getProperty(ApiKey, api_key_); - std::string username; - std::string password; - getProperty(Username, username); - getProperty(Password, password); + api_key_ = getProperty(ApiKey.name) | utils::toOptional(); + std::string username = getProperty(Username.name).value_or(""); + std::string password = getProperty(Password.name).value_or(""); + if (!username.empty() && !password.empty()) username_password_.emplace(std::move(username), std::move(password)); if (api_key_.has_value() == username_password_.has_value()) diff --git a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h index 3089c3f498..8ceab05406 100644 --- a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h +++ b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h @@ -46,6 +46,7 @@ class ElasticsearchCredentialsControllerService : public core::controller::Contr EXTENSIONAPI static constexpr auto ApiKey = core::PropertyDefinitionBuilder<>::createProperty("API Key") .withDescription("The API Key to use") .isSensitive(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ Username, diff --git a/extensions/elasticsearch/PostElasticsearch.cpp b/extensions/elasticsearch/PostElasticsearch.cpp index 6d9f8cf4f5..c68190f063 100644 --- a/extensions/elasticsearch/PostElasticsearch.cpp +++ b/extensions/elasticsearch/PostElasticsearch.cpp @@ -15,11 +15,10 @@ * limitations under the License. */ - #include "PostElasticsearch.h" -#include #include +#include #include "core/ProcessContext.h" #include "core/ProcessSession.h" @@ -27,8 +26,9 @@ #include "rapidjson/document.h" #include "rapidjson/stream.h" #include "rapidjson/writer.h" -#include "utils/expected.h" #include "utils/JsonCallback.h" +#include "utils/expected.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::extensions::elasticsearch { @@ -50,7 +50,7 @@ auto PostElasticsearch::getCredentialsService(core::ProcessContext& context) con } void PostElasticsearch::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - context.getProperty(MaxBatchSize, max_batch_size_); + max_batch_size_ = utils::parseU64Property(context, MaxBatchSize); if (max_batch_size_ < 1) throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Max Batch Size property is invalid"); @@ -96,7 +96,7 @@ class ElasticPayload { if (!index) return nonstd::make_unexpected("Missing index"); - auto id = context.getProperty(PostElasticsearch::Identifier, flow_file.get()); + auto id = context.getProperty(PostElasticsearch::Identifier, flow_file.get()) | utils::toOptional(); if (!id && (action == "delete" || action == "update" || action == "upsert")) return nonstd::make_unexpected("Identifier is required for DELETE,UPDATE and UPSERT actions"); diff --git a/extensions/elasticsearch/PostElasticsearch.h b/extensions/elasticsearch/PostElasticsearch.h index 5274bf25ec..e7427a0af2 100644 --- a/extensions/elasticsearch/PostElasticsearch.h +++ b/extensions/elasticsearch/PostElasticsearch.h @@ -50,7 +50,7 @@ class PostElasticsearch : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto MaxBatchSize = core::PropertyDefinitionBuilder<>::createProperty("Max Batch Size") .withDescription("The maximum number of flow files to process at a time.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("100") .build(); EXTENSIONAPI static constexpr auto ElasticCredentials = core::PropertyDefinitionBuilder<>::createProperty("Elasticsearch Credentials Provider Service") @@ -67,11 +67,13 @@ class PostElasticsearch : public core::ProcessorImpl { EXTENSIONAPI static constexpr auto Hosts = core::PropertyDefinitionBuilder<>::createProperty("Hosts") .withDescription("A comma-separated list of HTTP hosts that host Elasticsearch query nodes. Currently only supports a single host.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .isRequired(true) .build(); EXTENSIONAPI static constexpr auto Index = core::PropertyDefinitionBuilder<>::createProperty("Index") .withDescription("The name of the index to use.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .isRequired(true) .build(); EXTENSIONAPI static constexpr auto Identifier = core::PropertyDefinitionBuilder<>::createProperty("Identifier") @@ -79,6 +81,7 @@ class PostElasticsearch : public core::ProcessorImpl { "in which case the document's identifier will be auto-generated by Elasticsearch. " "For all other Actions, the attribute must evaluate to a non-empty value.") .supportsExpressionLanguage(true) + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ Action, diff --git a/extensions/execute-process/ExecuteProcess.cpp b/extensions/execute-process/ExecuteProcess.cpp index 21a215a2c4..060e1a15f5 100644 --- a/extensions/execute-process/ExecuteProcess.cpp +++ b/extensions/execute-process/ExecuteProcess.cpp @@ -33,6 +33,7 @@ #include "utils/Environment.h" #include "utils/StringUtils.h" #include "utils/gsl.h" +#include "utils/ProcessorConfigUtils.h" using namespace std::literals::chrono_literals; @@ -44,22 +45,20 @@ void ExecuteProcess::initialize() { } void ExecuteProcess::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - std::string value; - if (context.getProperty(Command, value)) { - command_ = value; + if (auto command = context.getProperty(Command)) { + command_ = *command; } - if (context.getProperty(CommandArguments, value)) { - command_argument_ = value; + if (auto const command_argument = context.getProperty(CommandArguments)) { + command_argument_ = *command_argument; } if (auto working_dir_str = context.getProperty(WorkingDir)) { working_dir_ = *working_dir_str; } - if (auto batch_duration = context.getProperty(BatchDuration)) { - batch_duration_ = batch_duration->getMilliseconds(); - logger_->log_debug("Setting batch duration to {}", batch_duration_); - } - if (context.getProperty(RedirectErrorStream, value)) { - redirect_error_stream_ = org::apache::nifi::minifi::utils::string::toBool(value).value_or(false); + batch_duration_ = utils::parseMsProperty(context, BatchDuration); + logger_->log_debug("Setting batch duration to {}", batch_duration_); + + if (const auto redirect_error_stream = utils::parseOptionalBoolProperty(context, RedirectErrorStream)) { + redirect_error_stream_ = *redirect_error_stream; } full_command_ = command_ + " " + command_argument_; } diff --git a/extensions/execute-process/ExecuteProcess.h b/extensions/execute-process/ExecuteProcess.h index 1992599691..31e1446940 100644 --- a/extensions/execute-process/ExecuteProcess.h +++ b/extensions/execute-process/ExecuteProcess.h @@ -74,12 +74,12 @@ class ExecuteProcess final : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto BatchDuration = core::PropertyDefinitionBuilder<>::createProperty("Batch Duration") .withDescription("If the process is expected to be long-running and produce textual output, a batch duration can be specified.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("0 sec") .build(); EXTENSIONAPI static constexpr auto RedirectErrorStream = core::PropertyDefinitionBuilder<>::createProperty("Redirect Error Stream") .withDescription("If true will redirect any error stream output of the process to the output stream.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto Properties = std::to_array({ diff --git a/extensions/execute-process/test/ExecuteProcessTests.cpp b/extensions/execute-process/test/ExecuteProcessTests.cpp index 0f993f53e9..d0838c6ea4 100644 --- a/extensions/execute-process/test/ExecuteProcessTests.cpp +++ b/extensions/execute-process/test/ExecuteProcessTests.cpp @@ -38,7 +38,7 @@ class ExecuteProcessTestsFixture { }; TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run a single command", "[ExecuteProcess]") { - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, "echo -n test")); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, "echo -n test")); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -53,8 +53,8 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run a single co TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run an executable with a parameter", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; std::string arguments = "0 test_data"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments, arguments)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments.name, arguments)); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -69,8 +69,8 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run an executab TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run an executable with escaped parameters", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; std::string arguments = R"(0 test_data test_data2 "test data 3" "\"test data 4\")"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments, arguments)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments.name, arguments)); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -84,7 +84,7 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can run an executab TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess does not produce a flowfile if no output is generated", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -95,8 +95,8 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess does not produce a TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can redirect error stream to stdout", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::RedirectErrorStream, "true")); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::RedirectErrorStream.name, "true")); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -111,9 +111,9 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can redirect error TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can change workdir", "[ExecuteProcess]") { auto command = "./EchoParameters"; std::string arguments = "0 test_data"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command)); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments, arguments)); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::WorkingDir, minifi::utils::file::get_executable_dir().string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments.name, arguments)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::WorkingDir.name, minifi::utils::file::get_executable_dir().string())); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -128,9 +128,9 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can change workdir" TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can forward long running output in batches", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; std::string arguments = "100 test_data1 test_data2"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments, arguments)); - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::BatchDuration, "10 ms")); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments.name, arguments)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::BatchDuration.name, "10 ms")); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); @@ -147,7 +147,7 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess can forward long ru TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess buffer long outputs", "[ExecuteProcess]") { auto command = minifi::utils::file::get_executable_dir() / "EchoParameters"; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command, command.string())); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::Command.name, command.string())); std::string param1; SECTION("Exact buffer size output") { @@ -158,7 +158,7 @@ TEST_CASE_METHOD(ExecuteProcessTestsFixture, "ExecuteProcess buffer long outputs } std::string arguments = "0 " + param1; - REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments, arguments)); + REQUIRE(execute_process_->setProperty(processors::ExecuteProcess::CommandArguments.name, arguments)); controller_.plan->scheduleProcessor(execute_process_); auto result = controller_.trigger(); diff --git a/extensions/expression-language/Expression.cpp b/extensions/expression-language/Expression.cpp index 3f53915dd5..cb3d03de3f 100644 --- a/extensions/expression-language/Expression.cpp +++ b/extensions/expression-language/Expression.cpp @@ -97,7 +97,7 @@ Expression make_dynamic_attr(const std::string &attribute_id) { if (cur_flow_file && cur_flow_file->getAttribute(attribute_id, result)) { return Value(result); } else { - auto registry = params.registry_.lock(); + auto registry = params.registry_; if (registry && registry->getConfigurationProperty(attribute_id , result)) { return Value(result); } diff --git a/extensions/expression-language/ExpressionContextBuilder.cpp b/extensions/expression-language/ExpressionContextBuilder.cpp index e8d7731dce..e033e6cdda 100644 --- a/extensions/expression-language/ExpressionContextBuilder.cpp +++ b/extensions/expression-language/ExpressionContextBuilder.cpp @@ -35,7 +35,7 @@ ExpressionContextBuilder::ExpressionContextBuilder(std::string_view name) ExpressionContextBuilder::~ExpressionContextBuilder() = default; -std::shared_ptr ExpressionContextBuilder::build(const std::shared_ptr &processor) { +std::shared_ptr ExpressionContextBuilder::build(Processor& processor) { return std::make_shared(processor, controller_service_provider_, prov_repo_, flow_repo_, configuration_, content_repo_); } diff --git a/extensions/expression-language/ExpressionContextBuilder.h b/extensions/expression-language/ExpressionContextBuilder.h index ef71d86d27..5f85ec0e81 100644 --- a/extensions/expression-language/ExpressionContextBuilder.h +++ b/extensions/expression-language/ExpressionContextBuilder.h @@ -44,7 +44,7 @@ class ExpressionContextBuilder : public core::ProcessContextBuilderImpl { EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; - std::shared_ptr build(const std::shared_ptr &processor) override; + std::shared_ptr build(Processor& processor) override; }; } // namespace org::apache::nifi::minifi::core::expressions diff --git a/extensions/expression-language/ProcessContextExpr.cpp b/extensions/expression-language/ProcessContextExpr.cpp index 922b9f7978..8a4665a059 100644 --- a/extensions/expression-language/ProcessContextExpr.cpp +++ b/extensions/expression-language/ProcessContextExpr.cpp @@ -16,67 +16,68 @@ */ #include "ProcessContextExpr.h" + #include #include +#include "asio/detail/mutex.hpp" +#include "utils/PropertyErrors.h" + namespace org::apache::nifi::minifi::core { -bool ProcessContextExpr::getProperty(bool supports_expression_language, std::string_view property_name, std::string& value, const FlowFile* const flow_file) { - if (!supports_expression_language) { - return ProcessContextImpl::getProperty(property_name, value); - } - std::string name{property_name}; - if (expressions_.find(name) == expressions_.end()) { - std::string expression_str; - if (!ProcessContextImpl::getProperty(name, expression_str)) { - return false; - } - logger_->log_debug("Compiling expression for {}/{}: {}", getProcessorNode()->getName(), name, expression_str); - expressions_.emplace(name, expression::compile(expression_str)); - expression_strs_.insert_or_assign(name, expression_str); +nonstd::expected ProcessContextExpr::getProperty(const std::string_view name, const FlowFile* flow_file) const { + const auto property = getProcessor().getPropertyReference(name); + if (!property) { + return nonstd::make_unexpected(PropertyErrorCode::NotSupportedProperty); } - minifi::expression::Parameters p(sharedFromThis(), flow_file); - value = expressions_[name](p).asString(); - logger_->log_debug(R"(expression "{}" of property "{}" evaluated to: {})", expression_strs_[name], name, value); - return true; -} - -bool ProcessContextExpr::getProperty(const Property& property, std::string& value, const FlowFile* const flow_file) { - return getProperty(property.supportsExpressionLanguage(), property.getName(), value, flow_file); -} - -bool ProcessContextExpr::getProperty(const PropertyReference& property, std::string& value, const FlowFile* const flow_file) { - return getProperty(property.supports_expression_language, property.name, value, flow_file); + if (!property->supports_expression_language) { + return ProcessContextImpl::getProperty(name, flow_file); + } + if (!cached_expressions_.contains(name)) { + auto expression_str = ProcessContextImpl::getProperty(name, flow_file); + if (!expression_str) { return expression_str; } + cached_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); + } + expression::Parameters p(this, flow_file); + auto result = cached_expressions_[std::string{name}](p).asString(); + if (!property->validator->validate(result)) { + return nonstd::make_unexpected(PropertyErrorCode::ValidationFailed); + } + return result; } -bool ProcessContextExpr::getDynamicProperty(const Property &property, std::string &value, const FlowFile* const flow_file) { - if (!property.supportsExpressionLanguage()) { - return ProcessContextImpl::getDynamicProperty(property.getName(), value); - } - auto name = property.getName(); - if (dynamic_property_expressions_.find(name) == dynamic_property_expressions_.end()) { - std::string expression_str; - ProcessContextImpl::getDynamicProperty(name, expression_str); - logger_->log_debug("Compiling expression for {}/{}: {}", getProcessorNode()->getName(), name, expression_str); - dynamic_property_expressions_.emplace(name, expression::compile(expression_str)); - expression_strs_.insert_or_assign(name, expression_str); +nonstd::expected ProcessContextExpr::getDynamicProperty(const std::string_view name, const FlowFile* flow_file) const { + if (!cached_dynamic_expressions_.contains(name)) { + auto expression_str = ProcessContextImpl::getDynamicProperty(name, flow_file); + if (!expression_str) { return expression_str; } + cached_dynamic_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); } - minifi::expression::Parameters p(sharedFromThis(), flow_file); - value = dynamic_property_expressions_[name](p).asString(); - logger_->log_debug(R"(expression "{}" of dynamic property "{}" evaluated to: {})", expression_strs_[name], name, value); - return true; + const expression::Parameters p(this, flow_file); + return cached_dynamic_expressions_[std::string{name}](p).asString(); } +nonstd::expected ProcessContextExpr::setProperty(const std::string_view name, std::string value) { + cached_expressions_.erase(std::string{name}); + return ProcessContextImpl::setProperty(name, std::move(value)); +} -bool ProcessContextExpr::setProperty(const std::string& property, std::string value) { - expressions_.erase(property); - return ProcessContextImpl::setProperty(property, value); +nonstd::expected ProcessContextExpr::setDynamicProperty(std::string name, std::string value) { + cached_dynamic_expressions_.erase(name); + return ProcessContextImpl::setDynamicProperty(std::move(name), std::move(value)); } -bool ProcessContextExpr::setDynamicProperty(const std::string& property, std::string value) { - dynamic_property_expressions_.erase(property); - return ProcessContextImpl::setDynamicProperty(property, value); +std::map ProcessContextExpr::getDynamicProperties(const FlowFile* flow_file) const { + auto dynamic_props = ProcessContextImpl::getDynamicProperties(flow_file); + for (auto& [dynamic_property_name, dynamic_property_value] : dynamic_props) { + if (!cached_dynamic_expressions_.contains(dynamic_property_name)) { + auto expression = expression::compile(dynamic_property_value); + expression::Parameters p(this, flow_file); + dynamic_property_value = expression(p).asString(); + cached_dynamic_expressions_.emplace(std::string{dynamic_property_name}, std::move(expression)); + } + } + return dynamic_props; } } // namespace org::apache::nifi::minifi::core diff --git a/extensions/expression-language/ProcessContextExpr.h b/extensions/expression-language/ProcessContextExpr.h index 43689892e8..fdd00caef4 100644 --- a/extensions/expression-language/ProcessContextExpr.h +++ b/extensions/expression-language/ProcessContextExpr.h @@ -34,14 +34,14 @@ namespace org::apache::nifi::minifi::core { */ class ProcessContextExpr final : public core::ProcessContextImpl { public: - ProcessContextExpr(const std::shared_ptr &processor, controller::ControllerServiceProvider* controller_service_provider, + ProcessContextExpr(Processor& processor, controller::ControllerServiceProvider* controller_service_provider, const std::shared_ptr &repo, const std::shared_ptr &flow_repo, const std::shared_ptr &content_repo = core::repository::createFileSystemRepository()) : core::ProcessContextImpl(processor, controller_service_provider, repo, flow_repo, content_repo), logger_(logging::LoggerFactory::getLogger()) { } - ProcessContextExpr(const std::shared_ptr &processor, controller::ControllerServiceProvider* controller_service_provider, + ProcessContextExpr(Processor& processor, controller::ControllerServiceProvider* controller_service_provider, const std::shared_ptr &repo, const std::shared_ptr &flow_repo, const std::shared_ptr &configuration, const std::shared_ptr &content_repo = core::repository::createFileSystemRepository()) : core::ProcessContextImpl(processor, controller_service_provider, repo, flow_repo, configuration, content_repo), @@ -50,24 +50,18 @@ class ProcessContextExpr final : public core::ProcessContextImpl { ~ProcessContextExpr() override = default; - bool getProperty(const Property& property, std::string &value, const FlowFile* const flow_file) override; + nonstd::expected getProperty(std::string_view name, const FlowFile*) const override; + nonstd::expected getDynamicProperty(std::string_view name, const FlowFile*) const override; - bool getProperty(const PropertyReference& property, std::string &value, const FlowFile* const flow_file) override; + nonstd::expected setProperty(std::string_view name, std::string value) override; + nonstd::expected setDynamicProperty(std::string name, std::string value) override; - bool getDynamicProperty(const Property &property, std::string &value, const FlowFile* const flow_file) override; - - bool setProperty(const std::string& property, std::string value) override; - - bool setDynamicProperty(const std::string& property, std::string value) override; - - using ProcessContextImpl::getProperty; + std::map getDynamicProperties(const FlowFile*) const override; private: - bool getProperty(bool supports_expression_language, std::string_view property_name, std::string& value, const FlowFile* const flow_file); + mutable std::unordered_map> cached_expressions_; + mutable std::unordered_map> cached_dynamic_expressions_; - std::unordered_map expressions_; - std::unordered_map dynamic_property_expressions_; - std::unordered_map expression_strs_; std::shared_ptr logger_; }; diff --git a/extensions/expression-language/impl/expression/Expression.h b/extensions/expression-language/impl/expression/Expression.h index cfe963730c..e87a1863ad 100644 --- a/extensions/expression-language/impl/expression/Expression.h +++ b/extensions/expression-language/impl/expression/Expression.h @@ -30,16 +30,14 @@ namespace org::apache::nifi::minifi::expression { struct Parameters { - const core::FlowFile* const flow_file; - std::weak_ptr registry_; - explicit Parameters(const std::shared_ptr& reg, const core::FlowFile* const ff = nullptr) + const core::FlowFile* const flow_file = nullptr; + const core::VariableRegistry* const registry_ = nullptr; + explicit Parameters(const core::VariableRegistry* const reg, const core::FlowFile* const ff = nullptr) : flow_file(ff), registry_(reg) { } - explicit Parameters(const core::FlowFile* const ff = nullptr) - : flow_file(ff) { - } + explicit Parameters(const core::FlowFile *const ff = nullptr) : flow_file(ff) {} }; class Expression; diff --git a/extensions/expression-language/tests/ProcessContextExprTests.cpp b/extensions/expression-language/tests/ProcessContextExprTests.cpp index 563961b1f9..00d7c2d409 100644 --- a/extensions/expression-language/tests/ProcessContextExprTests.cpp +++ b/extensions/expression-language/tests/ProcessContextExprTests.cpp @@ -99,37 +99,4 @@ TEST_CASE("ProcessContextExpr can update existing processor properties", "[setPr CHECK(context->getProperty(minifi::DummyProcessor::ExpressionLanguageProperty, nullptr) == "bar"); } } - - SECTION("Set and get simple dynamic property") { - static constexpr auto simple_property_definition = minifi::core::PropertyDefinitionBuilder<>::createProperty("Simple Dynamic Property") - .withDescription("A simple dynamic string property") - .build(); - core::Property simple_property{simple_property_definition}; - std::string property_value; - - context->setDynamicProperty(simple_property.getName(), "foo"); - CHECK(context->getDynamicProperty(simple_property, property_value, nullptr)); - CHECK(property_value == "foo"); - - context->setDynamicProperty(simple_property.getName(), "bar"); - CHECK(context->getDynamicProperty(simple_property, property_value, nullptr)); - CHECK(property_value == "bar"); - } - - SECTION("Set and get expression language dynamic property") { - static constexpr auto expression_language_property_definition = minifi::core::PropertyDefinitionBuilder<>::createProperty("Expression Language Dynamic Property") - .withDescription("A dynamic property which supports expression language") - .supportsExpressionLanguage(true) - .build(); - core::Property expression_language_property{expression_language_property_definition}; - std::string property_value; - - context->setDynamicProperty(expression_language_property.getName(), "foo"); - CHECK(context->getDynamicProperty(expression_language_property, property_value, nullptr)); - CHECK(property_value == "foo"); - - context->setDynamicProperty(expression_language_property.getName(), "bar"); - CHECK(context->getDynamicProperty(expression_language_property, property_value, nullptr)); - CHECK(property_value == "bar"); - } } diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp index 49a601ee96..eb9842fc45 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp @@ -40,13 +40,13 @@ std::shared_ptr GCPCredentialsControllerService::creat } std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonPath() const { - std::string json_path; - if (!getProperty(JsonFilePath, json_path)) { + const auto json_path = getProperty(JsonFilePath.name); + if (!json_path) { logger_->log_error("Missing or invalid {}", JsonFilePath.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(json_path); + auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(*json_path); if (!json_path_credentials.ok()) { logger_->log_error("{}", json_path_credentials.status().message()); return nullptr; @@ -55,13 +55,13 @@ std::shared_ptr GCPCredentialsControllerService::creat } std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonContents() const { - std::string json_contents; - if (!getProperty(JsonContents, json_contents)) { + auto json_contents = getProperty(JsonContents.name); + if (!json_contents) { logger_->log_error("Missing or invalid {}", JsonContents.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonContents(json_contents); + auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonContents(*json_contents); if (!json_path_credentials.ok()) { logger_->log_error("{}", json_path_credentials.status().message()); return nullptr; @@ -70,10 +70,9 @@ std::shared_ptr GCPCredentialsControllerService::creat } void GCPCredentialsControllerService::onEnable() { - std::string value; std::optional credentials_location; - if (getProperty(CredentialsLoc, value)) { - credentials_location = magic_enum::enum_cast(value); + if (const auto value = getProperty(CredentialsLoc.name)) { + credentials_location = magic_enum::enum_cast(*value); } if (!credentials_location) { logger_->log_error("Invalid Credentials Location, defaulting to {}", magic_enum::enum_name(CredentialsLocation::USE_DEFAULT_CREDENTIALS)); diff --git a/extensions/gcp/processors/DeleteGCSObject.cpp b/extensions/gcp/processors/DeleteGCSObject.cpp index a4dcadb395..bc4c01322a 100644 --- a/extensions/gcp/processors/DeleteGCSObject.cpp +++ b/extensions/gcp/processors/DeleteGCSObject.cpp @@ -17,11 +17,13 @@ #include "DeleteGCSObject.h" -#include "core/Resource.h" +#include "utils/ProcessorConfigUtils.h" + +#include "../GCPAttributes.h" +#include "core/FlowFile.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" -#include "core/FlowFile.h" -#include "../GCPAttributes.h" +#include "core/Resource.h" namespace gcs = ::google::cloud::storage; @@ -54,14 +56,11 @@ void DeleteGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSess } gcs::Generation generation; - - if (auto gen_str = context.getProperty(ObjectGeneration, flow_file.get()); gen_str && !gen_str->empty()) { - try { - int64_t gen = 0; - utils::internal::ValueParser(*gen_str).parse(gen).parseEnd(); - generation = gcs::Generation(gen); - } catch (const utils::internal::ValueException&) { - logger_->log_error("Invalid generation: {}", *gen_str); + if (const auto object_generation_str = context.getProperty(ObjectGeneration, flow_file.get()); object_generation_str && !object_generation_str->empty()) { + if (const auto geni64 = parsing::parseIntegral(*object_generation_str)) { + generation = gcs::Generation{*geni64}; + } else { + logger_->log_error("Invalid generation: {}", *object_generation_str); session.transfer(flow_file, Failure); return; } diff --git a/extensions/gcp/processors/FetchGCSObject.cpp b/extensions/gcp/processors/FetchGCSObject.cpp index 78b36f0c31..debd5bfefe 100644 --- a/extensions/gcp/processors/FetchGCSObject.cpp +++ b/extensions/gcp/processors/FetchGCSObject.cpp @@ -48,7 +48,7 @@ class FetchFromGCSCallback { if (!reader) return 0; std::string contents{std::istreambuf_iterator{reader}, {}}; - auto write_ret = stream->write(gsl::make_span(contents).as_span()); + const auto write_ret = gsl::narrow(stream->write(gsl::make_span(contents).as_span())); reader.Close(); return write_ret; } @@ -124,18 +124,19 @@ void FetchGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSessi FetchFromGCSCallback callback(client, *bucket, *object_name); callback.setEncryptionKey(encryption_key_); - if (auto gen_str = context.getProperty(ObjectGeneration, flow_file.get()); gen_str && !gen_str->empty()) { - try { - int64_t gen = 0; - utils::internal::ValueParser(*gen_str).parse(gen).parseEnd(); - callback.setGeneration(gcs::Generation(gen)); - } catch (const utils::internal::ValueException&) { - logger_->log_error("Invalid generation: {}", *gen_str); + gcs::Generation generation; + if (const auto object_generation_str = context.getProperty(ObjectGeneration, flow_file.get()); object_generation_str && !object_generation_str->empty()) { + if (const auto geni64 = parsing::parseIntegral(*object_generation_str)) { + generation = gcs::Generation{*geni64}; + } else { + logger_->log_error("Invalid generation: {}", *object_generation_str); session.transfer(flow_file, Failure); return; } } + callback.setGeneration(generation); + session.write(flow_file, std::ref(callback)); if (!callback.getStatus().ok()) { flow_file->setAttribute(GCS_STATUS_MESSAGE, callback.getStatus().message()); diff --git a/extensions/gcp/processors/GCSProcessor.cpp b/extensions/gcp/processors/GCSProcessor.cpp index c09206d246..2ae8e16d1a 100644 --- a/extensions/gcp/processors/GCSProcessor.cpp +++ b/extensions/gcp/processors/GCSProcessor.cpp @@ -16,17 +16,20 @@ */ #include "GCSProcessor.h" + +#include "utils/ProcessorConfigUtils.h" + +#include "../controllerservices/GCPCredentialsControllerService.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" -#include "../controllerservices/GCPCredentialsControllerService.h" namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { std::shared_ptr GCSProcessor::getCredentials(core::ProcessContext& context) const { - std::string service_name; - if (context.getProperty(GCSProcessor::GCPCredentials, service_name) && !IsNullOrEmpty(service_name)) { + const std::string service_name = utils::parseProperty(context, GCSProcessor::GCPCredentials); + if (!IsNullOrEmpty(service_name)) { auto gcp_credentials_controller_service = std::dynamic_pointer_cast(context.getControllerService(service_name, getUUID())); if (!gcp_credentials_controller_service) return nullptr; @@ -36,7 +39,7 @@ std::shared_ptr GCSProcessor::getCr } void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - if (auto number_of_retries = context.getProperty(NumberOfRetries)) { + if (auto number_of_retries = utils::parseOptionalU64Property(context, NumberOfRetries)) { retry_policy_ = std::make_shared(gsl::narrow(*number_of_retries)); } @@ -45,7 +48,7 @@ void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing GCP Credentials"); } - endpoint_url_ = context.getProperty(EndpointOverrideURL); + endpoint_url_ = context.getProperty(EndpointOverrideURL) | utils::toOptional(); if (endpoint_url_) logger_->log_debug("Endpoint overwritten: {}", *endpoint_url_); } diff --git a/extensions/gcp/processors/GCSProcessor.h b/extensions/gcp/processors/GCSProcessor.h index ff91633101..1fa7b62875 100644 --- a/extensions/gcp/processors/GCSProcessor.h +++ b/extensions/gcp/processors/GCSProcessor.h @@ -46,7 +46,7 @@ class GCSProcessor : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto NumberOfRetries = core::PropertyDefinitionBuilder<>::createProperty("Number of retries") .withDescription("How many retry attempts should be made before routing to the failure relationship.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("6") .isRequired(true) .supportsExpressionLanguage(false) diff --git a/extensions/gcp/processors/ListGCSBucket.cpp b/extensions/gcp/processors/ListGCSBucket.cpp index 8905b65179..77f2857659 100644 --- a/extensions/gcp/processors/ListGCSBucket.cpp +++ b/extensions/gcp/processors/ListGCSBucket.cpp @@ -17,11 +17,13 @@ #include "ListGCSBucket.h" -#include "core/Resource.h" +#include "utils/ProcessorConfigUtils.h" + +#include "../GCPAttributes.h" #include "core/FlowFile.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" -#include "../GCPAttributes.h" +#include "core/Resource.h" namespace gcs = ::google::cloud::storage; @@ -34,14 +36,14 @@ void ListGCSBucket::initialize() { void ListGCSBucket::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { GCSProcessor::onSchedule(context, session_factory); - context.getProperty(Bucket, bucket_); + bucket_ = utils::parseProperty(context, Bucket); } void ListGCSBucket::onTrigger(core::ProcessContext& context, core::ProcessSession& session) { gsl_Expects(gcp_credentials_); gcs::Client client = getClient(); - auto list_all_versions = context.getProperty(ListAllVersions); + auto list_all_versions = utils::parseOptionalBoolProperty(context, ListAllVersions); gcs::Versions versions = (list_all_versions && *list_all_versions) ? gcs::Versions(true) : gcs::Versions(false); auto objects_in_bucket = client.ListObjects(bucket_, versions); for (const auto& object_in_bucket : objects_in_bucket) { diff --git a/extensions/gcp/processors/ListGCSBucket.h b/extensions/gcp/processors/ListGCSBucket.h index 87dece09b3..13a4372946 100644 --- a/extensions/gcp/processors/ListGCSBucket.h +++ b/extensions/gcp/processors/ListGCSBucket.h @@ -58,7 +58,7 @@ class ListGCSBucket : public GCSProcessor { .build(); EXTENSIONAPI static constexpr auto ListAllVersions = core::PropertyDefinitionBuilder<>::createProperty("List all versions") .withDescription("Set this option to `true` to get all the previous versions separately.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("false") .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(GCSProcessor::Properties, std::to_array({ diff --git a/extensions/gcp/processors/PutGCSObject.cpp b/extensions/gcp/processors/PutGCSObject.cpp index b0218e3ddf..08b5b5746c 100644 --- a/extensions/gcp/processors/PutGCSObject.cpp +++ b/extensions/gcp/processors/PutGCSObject.cpp @@ -49,7 +49,7 @@ class UploadToGCSCallback { writer << content; writer.Close(); result_ = writer.metadata(); - return read_ret; + return gsl::narrow(read_ret); } [[nodiscard]] const google::cloud::StatusOr& getResult() const noexcept { @@ -158,7 +158,7 @@ void PutGCSObject::onTrigger(core::ProcessContext& context, core::ProcessSession if (auto predefined_acl = utils::parseOptionalEnumProperty(context, ObjectACL)) callback.setPredefinedAcl(*predefined_acl); - callback.setIfGenerationMatch(context.getProperty(OverwriteObject)); + callback.setIfGenerationMatch(utils::parseOptionalBoolProperty(context, OverwriteObject)); callback.setEncryptionKey(encryption_key_); diff --git a/extensions/gcp/processors/PutGCSObject.h b/extensions/gcp/processors/PutGCSObject.h index bdfe30fc19..f863a47a99 100644 --- a/extensions/gcp/processors/PutGCSObject.h +++ b/extensions/gcp/processors/PutGCSObject.h @@ -118,7 +118,7 @@ class PutGCSObject : public GCSProcessor { .build(); EXTENSIONAPI static constexpr auto OverwriteObject = core::PropertyDefinitionBuilder<>::createProperty("Overwrite Object") .withDescription("If false, the upload to GCS will succeed only if the object does not exist.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("true") .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(GCSProcessor::Properties, std::to_array({ diff --git a/extensions/gcp/tests/ListGCSBucketTests.cpp b/extensions/gcp/tests/ListGCSBucketTests.cpp index 8eac024c28..f0fbc3d570 100644 --- a/extensions/gcp/tests/ListGCSBucketTests.cpp +++ b/extensions/gcp/tests/ListGCSBucketTests.cpp @@ -81,7 +81,7 @@ class ListGCSBucketTests : public ::testing::Test { TEST_F(ListGCSBucketTests, MissingBucket) { EXPECT_CALL(*list_gcs_bucket_->mock_client_, CreateResumableUpload).Times(0); - EXPECT_THROW(test_controller_.trigger(), utils::internal::RequiredPropertyMissingException); + EXPECT_THROW(test_controller_.trigger(), std::runtime_error); } TEST_F(ListGCSBucketTests, ServerGivesPermaError) { diff --git a/extensions/grafana-loki/PushGrafanaLoki.cpp b/extensions/grafana-loki/PushGrafanaLoki.cpp index 191aade6ac..15ba811c57 100644 --- a/extensions/grafana-loki/PushGrafanaLoki.cpp +++ b/extensions/grafana-loki/PushGrafanaLoki.cpp @@ -130,15 +130,15 @@ void PushGrafanaLoki::onSchedule(core::ProcessContext& context, core::ProcessSes log_line_metadata_attributes_ = utils::string::splitAndTrimRemovingEmpty(*log_line_metadata_attributes, ","); } - auto log_line_batch_wait = context.getProperty(LogLineBatchWait); - auto log_line_batch_size = context.getProperty(LogLineBatchSize); + auto log_line_batch_wait = utils::parseOptionalMsProperty(context, LogLineBatchWait); + auto log_line_batch_size = utils::parseOptionalU64Property(context, LogLineBatchSize); if (log_line_batch_size && *log_line_batch_size < 1) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Log Line Batch Size property is missing or less than 1!"); } log_line_batch_size_is_set_ = log_line_batch_size.has_value(); log_line_batch_wait_is_set_ = log_line_batch_wait.has_value(); - max_batch_size_ = context.getProperty(MaxBatchSize); + max_batch_size_ = utils::parseOptionalU64Property(context, MaxBatchSize); if (max_batch_size_) { logger_->log_debug("PushGrafanaLoki Max Batch Size is set to: {}", *max_batch_size_); } @@ -149,8 +149,8 @@ void PushGrafanaLoki::onSchedule(core::ProcessContext& context, core::ProcessSes } if (log_line_batch_wait) { - log_batch_.setLogLineBatchWait(log_line_batch_wait->getMilliseconds()); - logger_->log_debug("PushGrafanaLoki Log Line Batch Wait is set to {} milliseconds", log_line_batch_wait->getMilliseconds()); + log_batch_.setLogLineBatchWait(*log_line_batch_wait); + logger_->log_debug("PushGrafanaLoki Log Line Batch Wait is set to {} milliseconds", *log_line_batch_wait); } } diff --git a/extensions/grafana-loki/PushGrafanaLoki.h b/extensions/grafana-loki/PushGrafanaLoki.h index 20148d0a25..9a6c76af10 100644 --- a/extensions/grafana-loki/PushGrafanaLoki.h +++ b/extensions/grafana-loki/PushGrafanaLoki.h @@ -54,25 +54,26 @@ class PushGrafanaLoki : public core::ProcessorImpl { .build(); EXTENSIONAPI static constexpr auto TenantID = core::PropertyDefinitionBuilder<>::createProperty("Tenant ID") .withDescription("The tenant ID used by default to push logs to Grafana Loki. If omitted or empty it assumes Grafana Loki is running in single-tenant mode and no X-Scope-OrgID header is sent.") + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto MaxBatchSize = core::PropertyDefinitionBuilder<>::createProperty("Max Batch Size") .withDescription("The maximum number of flow files to process at a time. If it is set to 0, all FlowFiles will be processed at once.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("100") .build(); EXTENSIONAPI static constexpr auto LogLineBatchWait = core::PropertyDefinitionBuilder<>::createProperty("Log Line Batch Wait") .withDescription("Time to wait before sending a log line batch to Grafana Loki, full or not. If this property and Log Line Batch Size are both unset, " "the log batch of the current trigger will be sent immediately.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto LogLineBatchSize = core::PropertyDefinitionBuilder<>::createProperty("Log Line Batch Size") .withDescription("Number of log lines to send in a batch to Loki. If this property and Log Line Batch Wait are both unset, " "the log batch of the current trigger will be sent immediately.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_INT_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto ConnectTimeout = core::PropertyDefinitionBuilder<>::createProperty("Connection Timeout") .withDescription("Max wait time for connection to the Grafana Loki service.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("5 s") .isRequired(true) .build(); diff --git a/extensions/grafana-loki/PushGrafanaLokiGrpc.cpp b/extensions/grafana-loki/PushGrafanaLokiGrpc.cpp index 4fe88202c3..1200f7db2a 100644 --- a/extensions/grafana-loki/PushGrafanaLokiGrpc.cpp +++ b/extensions/grafana-loki/PushGrafanaLokiGrpc.cpp @@ -56,17 +56,17 @@ void PushGrafanaLokiGrpc::setUpStreamLabels(core::ProcessContext& context) { void PushGrafanaLokiGrpc::setUpGrpcChannel(const std::string& url, core::ProcessContext& context) { ::grpc::ChannelArguments args; - if (auto keep_alive_time = context.getProperty(PushGrafanaLokiGrpc::KeepAliveTime)) { - logger_->log_debug("PushGrafanaLokiGrpc Keep Alive Time is set to {} ms", keep_alive_time->getMilliseconds().count()); - args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, gsl::narrow(keep_alive_time->getMilliseconds().count())); + if (auto keep_alive_time = utils::parseOptionalMsProperty(context, PushGrafanaLokiGrpc::KeepAliveTime)) { + logger_->log_debug("PushGrafanaLokiGrpc Keep Alive Time is set to {} ms", keep_alive_time->count()); + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, gsl::narrow(keep_alive_time->count())); } - if (auto keep_alive_timeout = context.getProperty(PushGrafanaLokiGrpc::KeepAliveTimeout)) { - logger_->log_debug("PushGrafanaLokiGrpc Keep Alive Timeout is set to {} ms", keep_alive_timeout->getMilliseconds().count()); - args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, gsl::narrow(keep_alive_timeout->getMilliseconds().count())); + if (auto keep_alive_timeout = utils::parseOptionalMsProperty(context, PushGrafanaLokiGrpc::KeepAliveTimeout)) { + logger_->log_debug("PushGrafanaLokiGrpc Keep Alive Timeout is set to {} ms", keep_alive_timeout->count()); + args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, gsl::narrow(keep_alive_timeout->count())); } - if (auto max_pings_without_data = context.getProperty(PushGrafanaLokiGrpc::MaxPingsWithoutData)) { + if (auto max_pings_without_data = utils::parseOptionalU64Property(context, PushGrafanaLokiGrpc::MaxPingsWithoutData)) { logger_->log_debug("PushGrafanaLokiGrpc Max Pings Without Data is set to {}", *max_pings_without_data); args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, gsl::narrow(*max_pings_without_data)); } @@ -94,13 +94,13 @@ void PushGrafanaLokiGrpc::setUpGrpcChannel(const std::string& url, core::Process void PushGrafanaLokiGrpc::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { PushGrafanaLoki::onSchedule(context, session_factory); - auto url = utils::getRequiredPropertyOrThrow(context, Url.name); + auto url = utils::parseProperty(context, Url); if (url.empty()) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Url property cannot be empty!"); } - tenant_id_ = context.getProperty(TenantID); - if (auto connection_timeout = context.getProperty(PushGrafanaLokiGrpc::ConnectTimeout)) { - connection_timeout_ms_ = connection_timeout->getMilliseconds(); + tenant_id_ = context.getProperty(TenantID) | utils::toOptional(); + if (auto connection_timeout = utils::parseOptionalMsProperty(context, PushGrafanaLokiGrpc::ConnectTimeout)) { + connection_timeout_ms_ = *connection_timeout; } else { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid connection timeout is set."); } diff --git a/extensions/grafana-loki/PushGrafanaLokiGrpc.h b/extensions/grafana-loki/PushGrafanaLokiGrpc.h index 78bc5f897d..178be68993 100644 --- a/extensions/grafana-loki/PushGrafanaLokiGrpc.h +++ b/extensions/grafana-loki/PushGrafanaLokiGrpc.h @@ -38,17 +38,17 @@ class PushGrafanaLokiGrpc final : public PushGrafanaLoki { EXTENSIONAPI static constexpr auto KeepAliveTime = core::PropertyDefinitionBuilder<>::createProperty("Keep Alive Time") .withDescription("The period after which a keepalive ping is sent on the transport. If not set, then the keep alive is disabled.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto KeepAliveTimeout = core::PropertyDefinitionBuilder<>::createProperty("Keep Alive Timeout") .withDescription("The amount of time the sender of the keepalive ping waits for an acknowledgement. If it does not receive an acknowledgment within this time, " "it will close the connection. If not set, then the default value 20 seconds is used.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto MaxPingsWithoutData = core::PropertyDefinitionBuilder<>::createProperty("Max Pings Without Data") .withDescription("The maximum number of pings that can be sent when there is no data/header frame to be sent. gRPC Core will not continue sending pings " "if we run over the limit. Setting it to 0 allows sending pings without such a restriction. If not set, then the default value 2 is used.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(PushGrafanaLoki::Properties, std::to_array({ KeepAliveTime, diff --git a/extensions/grafana-loki/PushGrafanaLokiREST.cpp b/extensions/grafana-loki/PushGrafanaLokiREST.cpp index 9f5470577d..d8cd25ee16 100644 --- a/extensions/grafana-loki/PushGrafanaLokiREST.cpp +++ b/extensions/grafana-loki/PushGrafanaLokiREST.cpp @@ -35,12 +35,12 @@ void PushGrafanaLokiREST::initialize() { } void PushGrafanaLokiREST::setupClientTimeouts(const core::ProcessContext& context) { - if (auto connection_timeout = context.getProperty(PushGrafanaLokiREST::ConnectTimeout)) { - client_.setConnectionTimeout(connection_timeout->getMilliseconds()); + if (auto connection_timeout = utils::parseOptionalMsProperty(context, PushGrafanaLokiREST::ConnectTimeout)) { + client_.setConnectionTimeout(*connection_timeout); } - if (auto read_timeout = context.getProperty(PushGrafanaLokiREST::ReadTimeout)) { - client_.setReadTimeout(read_timeout->getMilliseconds()); + if (auto read_timeout = utils::parseOptionalMsProperty(context, PushGrafanaLokiREST::ReadTimeout)) { + client_.setReadTimeout(*read_timeout); } } @@ -72,7 +72,7 @@ void PushGrafanaLokiREST::setAuthorization(const core::ProcessContext& context) } void PushGrafanaLokiREST::initializeHttpClient(core::ProcessContext& context) { - auto url = utils::getRequiredPropertyOrThrow(context, Url.name); + auto url = utils::parseProperty(context, Url); if (url.empty()) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Url property cannot be empty!"); } @@ -90,13 +90,7 @@ void PushGrafanaLokiREST::onSchedule(core::ProcessContext& context, core::Proces initializeHttpClient(context); client_.setContentType("application/json"); client_.setFollowRedirects(true); - - auto tenant_id = context.getProperty(TenantID); - if (tenant_id && !tenant_id->empty()) { - client_.setRequestHeader("X-Scope-OrgID", tenant_id); - } else { - client_.setRequestHeader("X-Scope-OrgID", std::nullopt); - } + client_.setRequestHeader("X-Scope-OrgID", context.getProperty(TenantID) | utils::toOptional()); setupClientTimeouts(context); setAuthorization(context); diff --git a/extensions/grafana-loki/PushGrafanaLokiREST.h b/extensions/grafana-loki/PushGrafanaLokiREST.h index 21757d8045..beb1fb7f7a 100644 --- a/extensions/grafana-loki/PushGrafanaLokiREST.h +++ b/extensions/grafana-loki/PushGrafanaLokiREST.h @@ -19,6 +19,7 @@ #include "PushGrafanaLoki.h" #include "http/HTTPClient.h" +#include "rapidjson/document.h" namespace org::apache::nifi::minifi::extensions::grafana::loki { @@ -34,7 +35,7 @@ class PushGrafanaLokiREST : public PushGrafanaLoki { EXTENSIONAPI static constexpr auto ReadTimeout = core::PropertyDefinitionBuilder<>::createProperty("Read Timeout") .withDescription("Max wait time for response from remote service.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("15 s") .isRequired(true) .build(); diff --git a/extensions/kafka/ConsumeKafka.cpp b/extensions/kafka/ConsumeKafka.cpp index f173cc3521..72e559a3de 100644 --- a/extensions/kafka/ConsumeKafka.cpp +++ b/extensions/kafka/ConsumeKafka.cpp @@ -32,10 +32,9 @@ namespace org::apache::nifi::minifi { namespace core { // The upper limit for Max Poll Time is 4 seconds. This is because Watchdog would potentially start // reporting issues with the processor health otherwise -ValidationResult ConsumeKafkaMaxPollTimePropertyType::validate(const std::string& subject, const std::string& input) const { - const auto parsed_value = utils::timeutils::StringToDuration(input); - const bool is_valid = parsed_value.has_value() && 0ms < *parsed_value && *parsed_value <= 4s; - return ValidationResult{.valid = is_valid, .subject = subject, .input = input}; +bool ConsumeKafkaMaxPollTimePropertyType::validate(const std::string_view input) const { + const auto parsed_time = parsing::parseDurationMinMax(input, 0ms, 4s); + return parsed_time.has_value(); } } // namespace core @@ -48,24 +47,23 @@ void ConsumeKafka::initialize() { void ConsumeKafka::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { // Required properties - kafka_brokers_ = utils::getRequiredPropertyOrThrow(context, KafkaBrokers.name); - topic_names_ = utils::listFromRequiredCommaSeparatedProperty(context, TopicNames.name); - topic_name_format_ = utils::getRequiredPropertyOrThrow(context, TopicNameFormat.name); - honor_transactions_ = utils::parseBooleanPropertyOrThrow(context, HonorTransactions.name); - group_id_ = utils::getRequiredPropertyOrThrow(context, GroupID.name); - offset_reset_ = utils::getRequiredPropertyOrThrow(context, OffsetReset.name); - key_attribute_encoding_ = utils::getRequiredPropertyOrThrow(context, KeyAttributeEncoding.name); - max_poll_time_milliseconds_ = utils::parseTimePropertyMSOrThrow(context, MaxPollTime.name); - session_timeout_milliseconds_ = utils::parseTimePropertyMSOrThrow(context, SessionTimeout.name); + kafka_brokers_ = utils::parseProperty(context, KafkaBrokers); + topic_names_ = utils::string::splitAndTrim(utils::parseProperty(context, TopicNames), ","); + topic_name_format_ = utils::parseProperty(context, TopicNameFormat); + honor_transactions_ = utils::parseBoolProperty(context, HonorTransactions); + group_id_ = utils::parseProperty(context, GroupID); + offset_reset_ = utils::parseProperty(context, OffsetReset); + key_attribute_encoding_ = utils::parseProperty(context, KeyAttributeEncoding); + max_poll_time_milliseconds_ = utils::parseMsProperty(context, MaxPollTime); + session_timeout_milliseconds_ = utils::parseMsProperty(context, SessionTimeout); // Optional properties - context.getProperty(MessageDemarcator, message_demarcator_); - context.getProperty(MessageHeaderEncoding, message_header_encoding_); - context.getProperty(DuplicateHeaderHandling, duplicate_header_handling_); + message_demarcator_ = context.getProperty(MessageDemarcator).value_or(""); + message_header_encoding_ = context.getProperty(MessageHeaderEncoding).value_or(""); + duplicate_header_handling_ = context.getProperty(DuplicateHeaderHandling).value_or(""); - headers_to_add_as_attributes_ = utils::listFromCommaSeparatedProperty(context, HeadersToAddAsAttributes.name); - max_poll_records_ = gsl::narrow(context.getProperty(MaxPollRecords) - .value_or(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE.parse(DEFAULT_MAX_POLL_RECORDS))); + headers_to_add_as_attributes_ = utils::string::splitAndTrim(utils::parseOptionalProperty(context, HeadersToAddAsAttributes).value_or(""), ","); + max_poll_records_ = gsl::narrow(utils::parseU64Property(context, MaxPollRecords)); if (!utils::string::equalsIgnoreCase(KEY_ATTR_ENCODING_UTF_8, key_attribute_encoding_) && !utils::string::equalsIgnoreCase(KEY_ATTR_ENCODING_HEX, key_attribute_encoding_)) { @@ -139,14 +137,12 @@ void ConsumeKafka::extend_config_from_dynamic_properties(const core::ProcessCont using utils::setKafkaConfigurationField; const std::vector dynamic_prop_keys = context.getDynamicPropertyKeys(); - if (dynamic_prop_keys.empty()) { return; } - logger_->log_info( - "Loading {} extra kafka configuration fields from ConsumeKafka dynamic " - "properties:", - dynamic_prop_keys.size()); - for (const std::string& key: dynamic_prop_keys) { - std::string value; - gsl_Expects(context.getDynamicProperty(key, value)); + if (dynamic_prop_keys.empty()) { + return; + } + logger_->log_info("Loading {} extra kafka configuration fields from ConsumeKafka dynamic properties:", dynamic_prop_keys.size()); + for (const std::string& key : dynamic_prop_keys) { + std::string value = context.getDynamicProperty(key) | utils::expect(""); logger_->log_info("{}: {}", key.c_str(), value.c_str()); setKafkaConfigurationField(*conf_, key, value); } diff --git a/extensions/kafka/ConsumeKafka.h b/extensions/kafka/ConsumeKafka.h index 79a99e0de0..a69a6935c6 100644 --- a/extensions/kafka/ConsumeKafka.h +++ b/extensions/kafka/ConsumeKafka.h @@ -23,26 +23,27 @@ #include #include -#include "KafkaConnection.h" #include "KafkaProcessorBase.h" +#include "core/logging/LoggerFactory.h" #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "core/PropertyType.h" #include "core/RelationshipDefinition.h" -#include "core/logging/LoggerFactory.h" #include "io/StreamPipe.h" #include "rdkafka.h" #include "rdkafka_utils.h" +#include "KafkaConnection.h" #include "utils/ArrayUtils.h" namespace org::apache::nifi::minifi { namespace core { -class ConsumeKafkaMaxPollTimePropertyType : public TimePeriodPropertyType { +class ConsumeKafkaMaxPollTimePropertyType final : public PropertyValidator { public: - constexpr ~ConsumeKafkaMaxPollTimePropertyType() override {} // NOLINT see comment at grandparent + constexpr ~ConsumeKafkaMaxPollTimePropertyType() override { } // NOLINT see comment at grandparent - [[nodiscard]] ValidationResult validate(const std::string& subject, const std::string& input) const override; + [[nodiscard]] bool validate(const std::string_view input) const override; + [[nodiscard]] std::optional getEquivalentNifiStandardValidatorName() const override { return std::nullopt; } }; inline constexpr ConsumeKafkaMaxPollTimePropertyType CONSUME_KAFKA_MAX_POLL_TIME_TYPE{}; @@ -79,8 +80,7 @@ class ConsumeKafka final : public KafkaProcessorBase { static constexpr std::string_view MSG_HEADER_COMMA_SEPARATED_MERGE = "Comma-separated Merge"; // Flowfile attributes written - static constexpr std::string_view KAFKA_COUNT_ATTR = - "kafka.count"; // Always 1 until we start supporting merging from batches + static constexpr std::string_view KAFKA_COUNT_ATTR = "kafka.count"; // Always 1 until we start supporting merging from batches static constexpr std::string_view KAFKA_MESSAGE_KEY_ATTR = "kafka.key"; static constexpr std::string_view KAFKA_OFFSET_ATTR = "kafka.offset"; static constexpr std::string_view KAFKA_PARTITION_ATTR = "kafka.partition"; @@ -89,161 +89,126 @@ class ConsumeKafka final : public KafkaProcessorBase { static constexpr std::string_view DEFAULT_MAX_POLL_RECORDS = "10000"; static constexpr std::string_view DEFAULT_MAX_POLL_TIME = "4 seconds"; - static constexpr const std::size_t METADATA_COMMUNICATIONS_TIMEOUT_MS{60000}; - - EXTENSIONAPI static constexpr const char* Description = - "Consumes messages from Apache Kafka and transform them into MiNiFi FlowFiles. " - "The application should make sure that the processor is triggered at regular intervals, even if no messages are " - "expected, to serve any queued callbacks waiting to be called. Rebalancing can also only happen on trigger."; - - EXTENSIONAPI static constexpr auto KafkaBrokers = - core::PropertyDefinitionBuilder<>::createProperty("Kafka Brokers") - .withDescription("A comma-separated list of known Kafka Brokers in the format :.") - .withPropertyType(core::StandardPropertyTypes::NON_BLANK_TYPE) - .withDefaultValue("localhost:9092") - .supportsExpressionLanguage(true) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto TopicNames = - core::PropertyDefinitionBuilder<>::createProperty("Topic Names") - .withDescription( - "The name of the Kafka Topic(s) to pull from. Multiple topic names are supported as a comma separated list.") - .supportsExpressionLanguage(true) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto TopicNameFormat = - core::PropertyDefinitionBuilder<2>::createProperty("Topic Name Format") - .withDescription( - "Specifies whether the Topic(s) provided are a comma separated list of names or a single regular expression. " - "Using regular expressions does not automatically discover Kafka topics created after the processor started.") - .withAllowedValues({TOPIC_FORMAT_NAMES, TOPIC_FORMAT_PATTERNS}) - .withDefaultValue(TOPIC_FORMAT_NAMES) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto HonorTransactions = - core::PropertyDefinitionBuilder<>::createProperty("Honor Transactions") - .withDescription( - "Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If " - "false, the Processor will use an \"isolation level\" of read_uncomitted. This means that messages will be " - "received as soon as they are written to Kafka but will be pulled, even if the producer cancels the " - "transactions. If this value is true, MiNiFi will not receive any messages for which the producer's " - "transaction was canceled, but this can result in some latency since the consumer must wait for the producer " - "to finish its entire transaction instead of pulling as the messages become available.") - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) - .withDefaultValue("true") - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto GroupID = - core::PropertyDefinitionBuilder<>::createProperty("Group ID") - .withDescription( - "A Group ID is used to identify consumers that are within the same consumer group. Corresponds to Kafka's " - "'group.id' property.") - .supportsExpressionLanguage(true) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto OffsetReset = - core::PropertyDefinitionBuilder<3>::createProperty("Offset Reset") - .withDescription( - "Allows you to manage the condition when there is no initial offset in Kafka or if the current offset does " - "not exist any more on the server (e.g. because that data has been deleted). Corresponds to Kafka's " - "'auto.offset.reset' property.") - .withAllowedValues({OFFSET_RESET_EARLIEST, OFFSET_RESET_LATEST, OFFSET_RESET_NONE}) - .withDefaultValue(OFFSET_RESET_LATEST) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto KeyAttributeEncoding = - core::PropertyDefinitionBuilder<2>::createProperty("Key Attribute Encoding") - .withDescription( - "FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the " - "attribute should be encoded.") - .withAllowedValues({KEY_ATTR_ENCODING_UTF_8, KEY_ATTR_ENCODING_HEX}) - .withDefaultValue(KEY_ATTR_ENCODING_UTF_8) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto MessageDemarcator = - core::PropertyDefinitionBuilder<>::createProperty("Message Demarcator") - .withDescription( - "Since KafkaConsumer receives messages in batches, you have an option to output FlowFiles which contains all " - "Kafka messages in a single batch for a given topic and partition and this property allows you to provide a " - "string (interpreted as UTF-8) to use for demarcating apart multiple Kafka messages. This is an optional " - "property and if not provided each Kafka message received will result in a single FlowFile which time it is " - "triggered. ") - .supportsExpressionLanguage(true) - .build(); - EXTENSIONAPI static constexpr auto MessageHeaderEncoding = - core::PropertyDefinitionBuilder<2>::createProperty("Message Header Encoding") - .withDescription( - "Any message header that is found on a Kafka message will be added to the outbound FlowFile as an attribute. " - "This property indicates the Character Encoding to use for deserializing the headers.") - .withAllowedValues({MSG_HEADER_ENCODING_UTF_8, MSG_HEADER_ENCODING_HEX}) - .withDefaultValue(MSG_HEADER_ENCODING_UTF_8) - .build(); - EXTENSIONAPI static constexpr auto HeadersToAddAsAttributes = - core::PropertyDefinitionBuilder<>::createProperty("Headers To Add As Attributes") - .withDescription( - "A comma separated list to match against all message headers. Any message header whose name matches an item " - "from the list will be added to the FlowFile as an Attribute. If not specified, no Header values will be " - "added as FlowFile attributes. The behaviour on when multiple headers of the same name are present is set " - "using the Duplicate Header Handling attribute.") - .build(); - EXTENSIONAPI static constexpr auto DuplicateHeaderHandling = - core::PropertyDefinitionBuilder<3>::createProperty("Duplicate Header Handling") - .withDescription( - "For headers to be added as attributes, this option specifies how to handle cases where multiple headers are " - "present with the same key. For example in case of receiving these two headers: \"Accept: text/html\" and " - "\"Accept: application/xml\" and we want to attach the value of \"Accept\" as a FlowFile attribute:\n - " - "\"Keep First\" attaches: \"Accept -> text/html\"\n - \"Keep Latest\" attaches: \"Accept -> " - "application/xml\"\n - \"Comma-separated Merge\" attaches: \"Accept -> text/html, application/xml\"\n") - .withAllowedValues({MSG_HEADER_KEEP_FIRST, MSG_HEADER_KEEP_LATEST, MSG_HEADER_COMMA_SEPARATED_MERGE}) - .withDefaultValue(MSG_HEADER_KEEP_LATEST) // Mirroring NiFi behaviour - .build(); - EXTENSIONAPI static constexpr auto MaxPollRecords = - core::PropertyDefinitionBuilder<>::createProperty("Max Poll Records") - .withDescription( - "Specifies the maximum number of records Kafka should return when polling each time the processor is " - "triggered.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) - .withDefaultValue(DEFAULT_MAX_POLL_RECORDS) - .build(); - EXTENSIONAPI static constexpr auto MaxPollTime = - core::PropertyDefinitionBuilder<>::createProperty("Max Poll Time") - .withDescription( - "Specifies the maximum amount of time the consumer can use for polling data from the brokers. Polling is a " - "blocking operation, so the upper limit of this value is specified in 4 seconds.") - .withPropertyType(core::CONSUME_KAFKA_MAX_POLL_TIME_TYPE) - .withDefaultValue(DEFAULT_MAX_POLL_TIME) - .isRequired(true) - .build(); - EXTENSIONAPI static constexpr auto SessionTimeout = - core::PropertyDefinitionBuilder<>::createProperty("Session Timeout") - .withDescription( - "Client group session and failure detection timeout. The consumer sends periodic heartbeats to indicate its " - "liveness to the broker. If no hearts are received by the broker for a group member within the session " - "timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is " - "configured with the broker configuration properties group.min.session.timeout.ms and " - "group.max.session.timeout.ms.") - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) - .withDefaultValue("60 seconds") - .build(); - EXTENSIONAPI static constexpr auto Properties = utils::array_cat(KafkaProcessorBase::Properties, - std::to_array({KafkaBrokers, - TopicNames, - TopicNameFormat, - HonorTransactions, - GroupID, - OffsetReset, - KeyAttributeEncoding, - MessageDemarcator, - MessageHeaderEncoding, - HeadersToAddAsAttributes, - DuplicateHeaderHandling, - MaxPollRecords, - MaxPollTime, - SessionTimeout})); + static constexpr const std::size_t METADATA_COMMUNICATIONS_TIMEOUT_MS{ 60000 }; + + EXTENSIONAPI static constexpr const char* Description = "Consumes messages from Apache Kafka and transform them into MiNiFi FlowFiles. " + "The application should make sure that the processor is triggered at regular intervals, even if no messages are expected, " + "to serve any queued callbacks waiting to be called. Rebalancing can also only happen on trigger."; + + EXTENSIONAPI static constexpr auto KafkaBrokers = core::PropertyDefinitionBuilder<>::createProperty("Kafka Brokers") + .withDescription("A comma-separated list of known Kafka Brokers in the format :.") + .withValidator(core::StandardPropertyTypes::NON_BLANK_VALIDATOR) + .withDefaultValue("localhost:9092") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto TopicNames = core::PropertyDefinitionBuilder<>::createProperty("Topic Names") + .withDescription("The name of the Kafka Topic(s) to pull from. Multiple topic names are supported as a comma separated list.") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto TopicNameFormat = core::PropertyDefinitionBuilder<2>::createProperty("Topic Name Format") + .withDescription("Specifies whether the Topic(s) provided are a comma separated list of names or a single regular expression. " + "Using regular expressions does not automatically discover Kafka topics created after the processor started.") + .withAllowedValues({TOPIC_FORMAT_NAMES, TOPIC_FORMAT_PATTERNS}) + .withDefaultValue(TOPIC_FORMAT_NAMES) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto HonorTransactions = core::PropertyDefinitionBuilder<>::createProperty("Honor Transactions") + .withDescription( + "Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If false, the Processor will use an \"isolation level\" of " + "read_uncomitted. This means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions. " + "If this value is true, MiNiFi will not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the consumer " + "must wait for the producer to finish its entire transaction instead of pulling as the messages become available.") + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) + .withDefaultValue("true") + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto GroupID = core::PropertyDefinitionBuilder<>::createProperty("Group ID") + .withDescription("A Group ID is used to identify consumers that are within the same consumer group. Corresponds to Kafka's 'group.id' property.") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto OffsetReset = core::PropertyDefinitionBuilder<3>::createProperty("Offset Reset") + .withDescription("Allows you to manage the condition when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that " + "data has been deleted). Corresponds to Kafka's 'auto.offset.reset' property.") + .withAllowedValues({OFFSET_RESET_EARLIEST, OFFSET_RESET_LATEST, OFFSET_RESET_NONE}) + .withDefaultValue(OFFSET_RESET_LATEST) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto KeyAttributeEncoding = core::PropertyDefinitionBuilder<2>::createProperty("Key Attribute Encoding") + .withDescription("FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the attribute should be encoded.") + .withAllowedValues({KEY_ATTR_ENCODING_UTF_8, KEY_ATTR_ENCODING_HEX}) + .withDefaultValue(KEY_ATTR_ENCODING_UTF_8) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto MessageDemarcator = core::PropertyDefinitionBuilder<>::createProperty("Message Demarcator") + .withDescription("Since KafkaConsumer receives messages in batches, you have an option to output FlowFiles which contains all Kafka messages in a single batch " + "for a given topic and partition and this property allows you to provide a string (interpreted as UTF-8) to use for demarcating apart multiple Kafka messages. " + "This is an optional property and if not provided each Kafka message received will result in a single FlowFile which time it is triggered. ") + .supportsExpressionLanguage(true) + .build(); + EXTENSIONAPI static constexpr auto MessageHeaderEncoding = core::PropertyDefinitionBuilder<2>::createProperty("Message Header Encoding") + .withDescription("Any message header that is found on a Kafka message will be added to the outbound FlowFile as an attribute. This property indicates the Character Encoding " + "to use for deserializing the headers.") + .withAllowedValues({MSG_HEADER_ENCODING_UTF_8, MSG_HEADER_ENCODING_HEX}) + .withDefaultValue(MSG_HEADER_ENCODING_UTF_8) + .build(); + EXTENSIONAPI static constexpr auto HeadersToAddAsAttributes = core::PropertyDefinitionBuilder<>::createProperty("Headers To Add As Attributes") + .withDescription("A comma separated list to match against all message headers. Any message header whose name matches an item from the list will be added to the FlowFile " + "as an Attribute. If not specified, no Header values will be added as FlowFile attributes. The behaviour on when multiple headers of the same name are present is set using " + "the Duplicate Header Handling attribute.") + .build(); + EXTENSIONAPI static constexpr auto DuplicateHeaderHandling = core::PropertyDefinitionBuilder<3>::createProperty("Duplicate Header Handling") + .withDescription("For headers to be added as attributes, this option specifies how to handle cases where multiple headers are present with the same key. " + "For example in case of receiving these two headers: \"Accept: text/html\" and \"Accept: application/xml\" and we want to attach the value of \"Accept\" " + "as a FlowFile attribute:\n" + " - \"Keep First\" attaches: \"Accept -> text/html\"\n" + " - \"Keep Latest\" attaches: \"Accept -> application/xml\"\n" + " - \"Comma-separated Merge\" attaches: \"Accept -> text/html, application/xml\"\n") + .withAllowedValues({MSG_HEADER_KEEP_FIRST, MSG_HEADER_KEEP_LATEST, MSG_HEADER_COMMA_SEPARATED_MERGE}) + .withDefaultValue(MSG_HEADER_KEEP_LATEST) // Mirroring NiFi behaviour + .build(); + EXTENSIONAPI static constexpr auto MaxPollRecords = core::PropertyDefinitionBuilder<>::createProperty("Max Poll Records") + .withDescription("Specifies the maximum number of records Kafka should return when polling each time the processor is triggered.") + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) + .withDefaultValue(DEFAULT_MAX_POLL_RECORDS) + .build(); + EXTENSIONAPI static constexpr auto MaxPollTime = core::PropertyDefinitionBuilder<>::createProperty("Max Poll Time") + .withDescription("Specifies the maximum amount of time the consumer can use for polling data from the brokers. " + "Polling is a blocking operation, so the upper limit of this value is specified in 4 seconds.") + .withValidator(core::CONSUME_KAFKA_MAX_POLL_TIME_TYPE) + .withDefaultValue(DEFAULT_MAX_POLL_TIME) + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto SessionTimeout = core::PropertyDefinitionBuilder<>::createProperty("Session Timeout") + .withDescription("Client group session and failure detection timeout. The consumer sends periodic heartbeats " + "to indicate its liveness to the broker. If no hearts are received by the broker for a group member within " + "the session timeout, the broker will remove the consumer from the group and trigger a rebalance. " + "The allowed range is configured with the broker configuration properties group.min.session.timeout.ms and group.max.session.timeout.ms.") + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) + .withDefaultValue("60 seconds") + .build(); + EXTENSIONAPI static constexpr auto Properties = utils::array_cat(KafkaProcessorBase::Properties, std::to_array({ + KafkaBrokers, + TopicNames, + TopicNameFormat, + HonorTransactions, + GroupID, + OffsetReset, + KeyAttributeEncoding, + MessageDemarcator, + MessageHeaderEncoding, + HeadersToAddAsAttributes, + DuplicateHeaderHandling, + MaxPollRecords, + MaxPollTime, + SessionTimeout + })); + EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", - "Incoming Kafka messages as flowfiles. Depending on the demarcation strategy, this can be one or multiple flowfiles " - "per message."}; + "Incoming Kafka messages as flowfiles. Depending on the demarcation strategy, this can be one or multiple flowfiles per message."}; EXTENSIONAPI static constexpr auto Relationships = std::array{Success}; EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true; @@ -253,8 +218,8 @@ class ConsumeKafka final : public KafkaProcessorBase { ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS - explicit ConsumeKafka(const std::string_view name, const utils::Identifier& uuid = utils::Identifier()) - : KafkaProcessorBase(name, uuid, core::logging::LoggerFactory::getLogger(uuid)) {} + explicit ConsumeKafka(std::string_view name, const utils::Identifier& uuid = utils::Identifier()) : + KafkaProcessorBase(name, uuid, core::logging::LoggerFactory::getLogger(uuid)) {} ConsumeKafka(const ConsumeKafka&) = delete; ConsumeKafka(ConsumeKafka&&) = delete; @@ -271,16 +236,14 @@ class ConsumeKafka final : public KafkaProcessorBase { void extend_config_from_dynamic_properties(const core::ProcessContext& context); void configure_new_connection(core::ProcessContext& context); static std::string extract_message(const rd_kafka_message_t& rkmessage); - std::vector poll_kafka_messages(); + std::vector> poll_kafka_messages(); utils::KafkaEncoding key_attr_encoding_attr_to_enum() const; utils::KafkaEncoding message_header_encoding_attr_to_enum() const; std::string resolve_duplicate_headers(const std::vector& matching_headers) const; std::vector get_matching_headers(const rd_kafka_message_t& message, const std::string& header_name) const; - std::vector> get_flowfile_attributes_from_message_header( - const rd_kafka_message_t& message) const; + std::vector> get_flowfile_attributes_from_message_header(const rd_kafka_message_t& message) const; void add_kafka_attributes_to_flowfile(core::FlowFile& flow_file, const rd_kafka_message_t& message) const; - std::optional>> transform_pending_messages_into_flowfiles( - core::ProcessSession& session) const; + std::optional>> transform_pending_messages_into_flowfiles(core::ProcessSession& session) const; void process_pending_messages(core::ProcessSession& session); std::string kafka_brokers_; @@ -298,13 +261,13 @@ class ConsumeKafka final : public KafkaProcessorBase { std::chrono::milliseconds max_poll_time_milliseconds_{}; std::chrono::milliseconds session_timeout_milliseconds_{}; - utils::rd_kafka_consumer_unique_ptr consumer_; - utils::rd_kafka_conf_unique_ptr conf_; - utils::rd_kafka_topic_partition_list_unique_ptr kf_topic_partition_list_; + std::unique_ptr consumer_; + std::unique_ptr conf_; + std::unique_ptr kf_topic_partition_list_; // Intermediate container type for messages that have been processed, but are - // not yet persisted (e.g. in case of I/O error) - std::vector pending_messages_; + // not yet persisted (eg. in case of I/O error) + std::vector> pending_messages_; std::mutex do_not_call_on_trigger_concurrently_; }; diff --git a/extensions/kafka/KafkaProcessorBase.cpp b/extensions/kafka/KafkaProcessorBase.cpp index 9eba09d9df..e4de864f1a 100644 --- a/extensions/kafka/KafkaProcessorBase.cpp +++ b/extensions/kafka/KafkaProcessorBase.cpp @@ -52,18 +52,17 @@ void KafkaProcessorBase::setKafkaAuthenticationParameters(core::ProcessContext& utils::setKafkaConfigurationField(*config, "sasl.mechanism", std::string{magic_enum::enum_name(sasl_mechanism)}); logger_->log_debug("Kafka sasl.mechanism [{}]", magic_enum::enum_name(sasl_mechanism)); - auto setKafkaConfigIfNotEmpty = - [this, &context, config](const core::PropertyReference& property, const std::string& kafka_config_name, bool log_value = true) { - std::string value; - if (context.getProperty(property, value) && !value.empty()) { - utils::setKafkaConfigurationField(*config, kafka_config_name, value); - if (log_value) { - logger_->log_debug("Kafka {} [{}]", kafka_config_name, value); - } else { - logger_->log_debug("Kafka {} was set", kafka_config_name); - } - } - }; + auto setKafkaConfigIfNotEmpty = [this, &context, config](const core::PropertyReference& property, const std::string& kafka_config_name, bool log_value = true) { + const std::string value = context.getProperty(property).value_or(""); + if (!value.empty()) { + utils::setKafkaConfigurationField(*config, kafka_config_name, value); + if (log_value) { + logger_->log_debug("Kafka {} [{}]", kafka_config_name, value); + } else { + logger_->log_debug("Kafka {} was set", kafka_config_name); + } + } + }; setKafkaConfigIfNotEmpty(KerberosServiceName, "sasl.kerberos.service.name"); setKafkaConfigIfNotEmpty(KerberosPrincipal, "sasl.kerberos.principal"); diff --git a/extensions/kafka/PublishKafka.cpp b/extensions/kafka/PublishKafka.cpp index 894ef9d1bf..5fceaf13ca 100644 --- a/extensions/kafka/PublishKafka.cpp +++ b/extensions/kafka/PublishKafka.cpp @@ -335,35 +335,26 @@ void PublishKafka::onSchedule(core::ProcessContext& context, core::ProcessSessio interrupted_ = false; // Try to get a KafkaConnection - std::string client_id; - std::string brokers; - if (!context.getProperty(ClientName, client_id, nullptr)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Client Name property missing or invalid"); - } - if (!context.getProperty(SeedBrokers, brokers, nullptr)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Known Brokers property missing or invalid"); - } - + std::string client_id = utils::parseProperty(context, ClientName); + std::string brokers = utils::parseProperty(context, SeedBrokers); // Get some properties not (only) used directly to set up librdkafka // Batch Size - context.getProperty(BatchSize, batch_size_); + batch_size_ = utils::parseU64Property(context, BatchSize); logger_->log_debug("PublishKafka: Batch Size [{}]", batch_size_); // Target Batch Payload Size - context.getProperty(TargetBatchPayloadSize, target_batch_payload_size_); + target_batch_payload_size_ = utils::parseDataSizeProperty(context, TargetBatchPayloadSize); logger_->log_debug("PublishKafka: Target Batch Payload Size [{}]", target_batch_payload_size_); // Max Flow Segment Size - context.getProperty(MaxFlowSegSize, max_flow_seg_size_); + max_flow_seg_size_ = utils::parseDataSizeProperty(context, MaxFlowSegSize); logger_->log_debug("PublishKafka: Max Flow Segment Size [{}]", max_flow_seg_size_); // Attributes to Send as Headers - if (const auto attribute_name_regex = context.getProperty(AttributeNameRegex); - attribute_name_regex && !attribute_name_regex->empty()) { - attributeNameRegex_ = utils::Regex(*attribute_name_regex); - logger_->log_debug("PublishKafka: AttributeNameRegex [{}]", *attribute_name_regex); - } + attributeNameRegex_ = context.getProperty(AttributeNameRegex) + | utils::transform([](const auto pattern_str) { return utils::Regex{pattern_str}; }) + | utils::toOptional(); key_.brokers_ = brokers; key_.client_id_ = client_id; @@ -371,12 +362,8 @@ void PublishKafka::onSchedule(core::ProcessContext& context, core::ProcessSessio conn_ = std::make_unique(key_); configureNewConnection(context); - if (const auto message_key_field = context.getProperty(MessageKeyField); - message_key_field && !message_key_field->empty()) { - logger_->log_error( - "The {} property is set. This property is DEPRECATED and has no " - "effect; please use Kafka Key instead.", - MessageKeyField.name); + if (const auto message_key_field = context.getProperty(MessageKeyField); message_key_field && !message_key_field->empty()) { + logger_->log_error("The {} property is set. This property is DEPRECATED and has no effect; please use Kafka Key instead.", MessageKeyField.name); } logger_->log_debug("Successfully configured PublishKafka"); @@ -425,94 +412,72 @@ bool PublishKafka::configureNewConnection(core::ProcessContext& context) { throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } - if (auto debug_contexts = context.getProperty(DebugContexts); debug_contexts && !debug_contexts->empty()) { - result = rd_kafka_conf_set(conf_.get(), "debug", debug_contexts->c_str(), err_chars.data(), err_chars.size()); - logger_->log_debug("PublishKafka: debug [{}]", *debug_contexts); + if (const auto debug_context = context.getProperty(DebugContexts)) { + result = rd_kafka_conf_set(conf_.get(), "debug", debug_context->c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: debug [{}]", *debug_context); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto max_message_size = context.getProperty(MaxMessageSize); max_message_size && !max_message_size->empty()) { - result = rd_kafka_conf_set(conf_.get(), - "message.max.bytes", - max_message_size->c_str(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: message.max.bytes [{}]", max_message_size); + if (const auto max_message_size = context.getProperty(MaxMessageSize); max_message_size && !max_message_size->empty()) { + result = rd_kafka_conf_set(conf_.get(), "message.max.bytes", max_message_size->c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: message.max.bytes [{}]", *max_message_size); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto queue_buffer_max_message = context.getProperty(QueueBufferMaxMessage)) { + if (const auto queue_buffer_max_message = utils::parseOptionalU64Property(context, QueueBufferMaxMessage)) { if (*queue_buffer_max_message < batch_size_) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, - "Invalid configuration: Batch Size cannot be larger than Queue Max " - "Message"); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid configuration: Batch Size cannot be larger than Queue Max Message"); } - auto queue_buffer_max_message_str = std::to_string(*queue_buffer_max_message); - result = rd_kafka_conf_set(conf_.get(), - "queue.buffering.max.messages", - queue_buffer_max_message_str.c_str(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: queue.buffering.max.messages [{}]", queue_buffer_max_message_str); + const auto value = std::to_string(*queue_buffer_max_message); + result = rd_kafka_conf_set(conf_.get(), "queue.buffering.max.messages", value.c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: queue.buffering.max.messages [{}]", value); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto queue_buffer_max_size = context.getProperty(QueueBufferMaxSize)) { - auto queue_buffer_max_size_kb = queue_buffer_max_size->getValue() / 1024; - auto queue_buffer_max_size_kb_str = std::to_string(queue_buffer_max_size_kb); - result = rd_kafka_conf_set(conf_.get(), - "queue.buffering.max.kbytes", - queue_buffer_max_size_kb_str.c_str(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: queue.buffering.max.kbytes [{}]", queue_buffer_max_size_kb_str); + if (const auto queue_buffer_max_size = utils::parseOptionalDataSizeProperty(context, QueueBufferMaxSize)) { + auto valInt = *queue_buffer_max_size / 1024; + auto valueConf = std::to_string(valInt); + result = rd_kafka_conf_set(conf_.get(), "queue.buffering.max.kbytes", valueConf.c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: queue.buffering.max.kbytes [{}]", valueConf); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto queue_buffer_max_time = context.getProperty(QueueBufferMaxTime)) { - auto queue_buffer_max_time_ms_str = std::to_string(queue_buffer_max_time->getMilliseconds().count()); - result = rd_kafka_conf_set(conf_.get(), - "queue.buffering.max.ms", - queue_buffer_max_time_ms_str.c_str(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: queue.buffering.max.ms [{}]", queue_buffer_max_time_ms_str); + if (const auto queue_buffer_max_time = utils::parseOptionalMsProperty(context, QueueBufferMaxTime)) { + auto valueConf = std::to_string(queue_buffer_max_time->count()); + result = rd_kafka_conf_set(conf_.get(), "queue.buffering.max.ms", valueConf.c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: queue.buffering.max.ms [{}]", valueConf); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto batch_size = context.getProperty(BatchSize); batch_size && !batch_size->empty()) { - result = rd_kafka_conf_set(conf_.get(), "batch.num.messages", batch_size->c_str(), err_chars.data(), err_chars.size()); - logger_->log_debug("PublishKafka: batch.num.messages [{}]", *batch_size); + if (const auto batch_size = utils::parseOptionalU64Property(context, BatchSize)) { + auto value = std::to_string(*batch_size); + result = rd_kafka_conf_set(conf_.get(), "batch.num.messages", value.c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: batch.num.messages [{}]", value); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); } } - if (auto compress_codec = utils::parseOptionalEnumProperty(context, CompressCodec)) { - auto compress_codec_str = magic_enum::enum_name(*compress_codec); - result = rd_kafka_conf_set(conf_.get(), - "compression.codec", - compress_codec_str.data(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: compression.codec [{}]", compress_codec_str); + if (const auto compress_codec = context.getProperty(CompressCodec); compress_codec && !compress_codec->empty() && *compress_codec != "none") { + result = rd_kafka_conf_set(conf_.get(), "compression.codec", compress_codec->c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: compression.codec [{}]", *compress_codec); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); @@ -525,18 +490,10 @@ bool PublishKafka::configureNewConnection(core::ProcessContext& context) { const auto& dynamic_prop_keys = context.getDynamicPropertyKeys(); logger_->log_info("PublishKafka registering {} librdkafka dynamic properties", dynamic_prop_keys.size()); - for (const auto& prop_key: dynamic_prop_keys) { - core::Property dynamic_property_key{prop_key, "dynamic property"}; - dynamic_property_key.setSupportsExpressionLanguage(true); - std::string dynamic_property_value; - if (context.getDynamicProperty(dynamic_property_key, dynamic_property_value, nullptr) && - !dynamic_property_value.empty()) { - logger_->log_debug("PublishKafka: DynamicProperty: [{}] -> [{}]", prop_key, dynamic_property_value); - result = rd_kafka_conf_set(conf_.get(), - prop_key.c_str(), - dynamic_property_value.c_str(), - err_chars.data(), - err_chars.size()); + for (const auto& prop_key : dynamic_prop_keys) { + if (const auto dynamic_property_value = context.getDynamicProperty(prop_key, nullptr); dynamic_property_value && !dynamic_property_value->empty()) { + logger_->log_debug("PublishKafka: DynamicProperty: [{}] -> [{}]", prop_key, *dynamic_property_value); + result = rd_kafka_conf_set(conf_.get(), prop_key.c_str(), dynamic_property_value->c_str(), err_chars.data(), err_chars.size()); if (result != RD_KAFKA_CONF_OK) { auto error_msg = utils::string::join_pack(PREFIX_ERROR_MSG, err_chars.data()); throw Exception(PROCESS_SCHEDULE_EXCEPTION, error_msg); @@ -577,12 +534,9 @@ bool PublishKafka::createNewTopic(core::ProcessContext& context, const std::stri } rd_kafka_conf_res_t result = RD_KAFKA_CONF_OK; - std::string value; std::array err_chars{}; - std::string valueConf; - value = ""; - if (context.getProperty(DeliveryGuarantee, value, flow_file.get()) && !value.empty()) { + if (auto delivery_guarantee = context.getProperty(DeliveryGuarantee, flow_file.get())) { /* * Because of a previous error in this processor, the default value of this property was "DELIVERY_ONE_NODE". * As this is not a valid value for "request.required.acks", the following rd_kafka_topic_conf_set call failed, @@ -592,35 +546,23 @@ bool PublishKafka::createNewTopic(core::ProcessContext& context, const std::stri * of just one. In order not to break configurations generated with earlier versions and keep the same behaviour * as they had, we have to map "DELIVERY_ONE_NODE" to "-1" here. */ - if (value == "DELIVERY_ONE_NODE") { - value = "-1"; - logger_->log_warn( - "Using DELIVERY_ONE_NODE as the Delivery Guarantee property is " - "deprecated and is translated to -1 " - "(block until message is committed by all in sync replicas) for " - "backwards compatibility. " - "If you want to wait for one acknowledgment use '1' as the " - "property."); + if (*delivery_guarantee == "DELIVERY_ONE_NODE") { + delivery_guarantee = "-1"; + logger_->log_warn("Using DELIVERY_ONE_NODE as the Delivery Guarantee property is deprecated and is translated to -1 " + "(block until message is committed by all in sync replicas) for backwards compatibility. " + "If you want to wait for one acknowledgment use '1' as the property."); } - result = rd_kafka_topic_conf_set(topic_conf_.get(), - "request.required.acks", - value.c_str(), - err_chars.data(), - err_chars.size()); - logger_->log_debug("PublishKafka: request.required.acks [{}]", value); + result = rd_kafka_topic_conf_set(topic_conf_.get(), "request.required.acks", delivery_guarantee->c_str(), err_chars.data(), err_chars.size()); + logger_->log_debug("PublishKafka: request.required.acks [{}]", *delivery_guarantee); if (result != RD_KAFKA_CONF_OK) { logger_->log_error("PublishKafka: configure request.required.acks error result [{}]", err_chars.data()); return false; } } - if (const auto request_timeout = context.getProperty(RequestTimeOut)) { - valueConf = std::to_string(request_timeout->getMilliseconds().count()); - result = rd_kafka_topic_conf_set(topic_conf_.get(), - "request.timeout.ms", - valueConf.c_str(), - err_chars.data(), - err_chars.size()); + if (const auto request_timeout = utils::parseOptionalMsProperty(context, RequestTimeOut)) { + auto valueConf = std::to_string(request_timeout->count()); + result = rd_kafka_topic_conf_set(topic_conf_.get(), "request.timeout.ms", valueConf.c_str(), err_chars.data(), err_chars.size()); logger_->log_debug("PublishKafka: request.timeout.ms [{}]", valueConf); if (result != RD_KAFKA_CONF_OK) { logger_->log_error("PublishKafka: configure request.timeout.ms error result [{}]", err_chars.data()); @@ -628,13 +570,9 @@ bool PublishKafka::createNewTopic(core::ProcessContext& context, const std::stri } } - if (const auto message_timeout = context.getProperty(MessageTimeOut)) { - valueConf = std::to_string(message_timeout->getMilliseconds().count()); - result = rd_kafka_topic_conf_set(topic_conf_.get(), - "message.timeout.ms", - valueConf.c_str(), - err_chars.data(), - err_chars.size()); + if (const auto message_timeout = utils::parseOptionalMsProperty(context, MessageTimeOut)) { + auto valueConf = std::to_string(message_timeout->count()); + result = rd_kafka_topic_conf_set(topic_conf_.get(), "message.timeout.ms", valueConf.c_str(), err_chars.data(), err_chars.size()); logger_->log_debug("PublishKafka: message.timeout.ms [{}]", valueConf); if (result != RD_KAFKA_CONF_OK) { logger_->log_error("PublishKafka: configure message.timeout.ms error result [{}]", err_chars.data()); @@ -685,7 +623,7 @@ void PublishKafka::onTrigger(core::ProcessContext& context, core::ProcessSession // Collect FlowFiles to process uint64_t actual_bytes = 0U; std::vector> flowFiles; - for (uint32_t i = 0; i < batch_size_; i++) { + for (uint64_t i = 0; i < batch_size_; i++) { std::shared_ptr flowFile = session.get(); if (flowFile == nullptr) { break; } actual_bytes += flowFile->getSize(); @@ -715,9 +653,9 @@ void PublishKafka::onTrigger(core::ProcessContext& context, core::ProcessSession const size_t flow_file_index = messages->addFlowFile(); // Get Topic (FlowFile-dependent EL property) - std::string topic; - if (context.getProperty(Topic, topic, flowFile.get())) { - logger_->log_debug("PublishKafka: topic for flow file {} is '{}'", flowFile->getUUIDStr(), topic); + const auto topic = context.getProperty(Topic, flowFile.get()); + if (topic) { + logger_->log_debug("PublishKafka: topic for flow file {} is '{}'", flowFile->getUUIDStr(), *topic); } else { logger_->log_error("Flow file {} does not have a valid Topic", flowFile->getUUIDStr()); messages->modifyResult(flow_file_index, @@ -726,29 +664,30 @@ void PublishKafka::onTrigger(core::ProcessContext& context, core::ProcessSession } // Add topic to the connection if needed - if (!conn_->hasTopic(topic)) { - if (!createNewTopic(context, topic, flowFile)) { - logger_->log_error("Failed to add topic {}", topic); - messages->modifyResult(flow_file_index, - [](FlowFileResult& flow_file_result) { flow_file_result.flow_file_error = true; }); + if (!conn_->hasTopic(*topic)) { + if (!createNewTopic(context, *topic, flowFile)) { + logger_->log_error("Failed to add topic {}", *topic); + messages->modifyResult(flow_file_index, [](FlowFileResult& flow_file_result) { + flow_file_result.flow_file_error = true; + }); continue; } } - std::string kafkaKey; - if (!context.getProperty(KafkaKey, kafkaKey, flowFile.get()) || kafkaKey.empty()) { kafkaKey = flowFile->getUUIDStr(); } + std::string kafkaKey = context.getProperty(KafkaKey, flowFile.get()).value_or(flowFile->getUUIDStr()); + logger_->log_debug("PublishKafka: Message Key [{}]", kafkaKey); - auto thisTopic = conn_->getTopic(topic); + auto thisTopic = conn_->getTopic(*topic); if (thisTopic == nullptr) { - logger_->log_error("Topic {} is invalid", topic); - messages->modifyResult(flow_file_index, - [](FlowFileResult& flow_file_result) { flow_file_result.flow_file_error = true; }); + logger_->log_error("Topic {} is invalid", *topic); + messages->modifyResult(flow_file_index, [](FlowFileResult& flow_file_result) { + flow_file_result.flow_file_error = true; + }); continue; } - bool failEmptyFlowFiles = true; - context.getProperty(FailEmptyFlowFiles, failEmptyFlowFiles); + bool failEmptyFlowFiles = utils::parseBoolProperty(context, FailEmptyFlowFiles); ReadCallback callback(max_flow_seg_size_, kafkaKey, @@ -777,9 +716,10 @@ void PublishKafka::onTrigger(core::ProcessContext& context, core::ProcessSession } if (callback.status_ < 0) { - logger_->log_error("Failed to send flow to kafka topic {}, error: {}", topic, callback.error_); - messages->modifyResult(flow_file_index, - [](FlowFileResult& flow_file_result) { flow_file_result.flow_file_error = true; }); + logger_->log_error("Failed to send flow to kafka topic {}, error: {}", *topic, callback.error_); + messages->modifyResult(flow_file_index, [](FlowFileResult& flow_file_result) { + flow_file_result.flow_file_error = true; + }); continue; } } diff --git a/extensions/kafka/PublishKafka.h b/extensions/kafka/PublishKafka.h index 5ddf035b26..c3c4ee2aa7 100644 --- a/extensions/kafka/PublishKafka.h +++ b/extensions/kafka/PublishKafka.h @@ -92,14 +92,14 @@ class PublishKafka final : public KafkaProcessorBase { core::PropertyDefinitionBuilder<>::createProperty("Request Timeout") .withDescription("The ack timeout of the producer request") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("10 sec") .build(); EXTENSIONAPI static constexpr auto MessageTimeOut = core::PropertyDefinitionBuilder<>::createProperty("Message Timeout") .withDescription("The total time sending a message could take") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("30 sec") .build(); EXTENSIONAPI static constexpr auto ClientName = @@ -112,7 +112,7 @@ class PublishKafka final : public KafkaProcessorBase { core::PropertyDefinitionBuilder<>::createProperty("Batch Size") .withDescription("Maximum number of messages batched in one MessageSet") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_INT_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("10") .build(); EXTENSIONAPI static constexpr auto TargetBatchPayloadSize = @@ -121,7 +121,7 @@ class PublishKafka final : public KafkaProcessorBase { "The target total payload size for a batch. 0 B means unlimited " "(Batch Size is still applied).") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withValidator(core::StandardPropertyTypes::DATA_SIZE_VALIDATOR) .withDefaultValue("512 KB") .build(); EXTENSIONAPI static constexpr auto AttributeNameRegex = @@ -136,21 +136,21 @@ class PublishKafka final : public KafkaProcessorBase { "Delay to wait for messages in the producer queue to accumulate " "before constructing message batches") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withValidator(core::StandardPropertyTypes::TIME_PERIOD_VALIDATOR) .withDefaultValue("5 millis") .build(); EXTENSIONAPI static constexpr auto QueueBufferMaxSize = core::PropertyDefinitionBuilder<>::createProperty("Queue Max Buffer Size") .withDescription("Maximum total message size sum allowed on the producer queue") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withValidator(core::StandardPropertyTypes::DATA_SIZE_VALIDATOR) .withDefaultValue("1 MB") .build(); EXTENSIONAPI static constexpr auto QueueBufferMaxMessage = core::PropertyDefinitionBuilder<>::createProperty("Queue Max Message") .withDescription("Maximum number of messages allowed on the producer queue") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("1000") .build(); EXTENSIONAPI static constexpr auto CompressCodec = @@ -166,7 +166,7 @@ class PublishKafka final : public KafkaProcessorBase { "Maximum flow content payload segment size for the kafka record. " "0 B means unlimited.") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withValidator(core::StandardPropertyTypes::DATA_SIZE_VALIDATOR) .withDefaultValue("0 B") .build(); EXTENSIONAPI static constexpr auto SecurityCA = @@ -223,7 +223,7 @@ class PublishKafka final : public KafkaProcessorBase { "to failure. The old behavior is " "deprecated. Use connections to drop empty flow files!") .isRequired(false) - .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withValidator(core::StandardPropertyTypes::BOOLEAN_VALIDATOR) .withDefaultValue("true") .build(); EXTENSIONAPI static constexpr auto Properties = utils::array_cat(KafkaProcessorBase::Properties, @@ -274,7 +274,7 @@ class PublishKafka final : public KafkaProcessorBase { std::unique_ptr conn_; std::mutex connection_mutex_; - uint32_t batch_size_{}; + uint64_t batch_size_{}; uint64_t target_batch_payload_size_{}; uint64_t max_flow_seg_size_{}; std::optional attributeNameRegex_; diff --git a/extensions/kafka/tests/PublishKafkaTests.cpp b/extensions/kafka/tests/PublishKafkaTests.cpp index dbefd9c391..29a387d9bf 100644 --- a/extensions/kafka/tests/PublishKafkaTests.cpp +++ b/extensions/kafka/tests/PublishKafkaTests.cpp @@ -27,44 +27,43 @@ TEST_CASE("Scheduling should fail when batch size is larger than the max queue m LogTestController::getInstance().setTrace(); SingleProcessorTestController test_controller(std::make_unique("PublishKafka")); const auto publish_kafka = test_controller.getProcessor(); - publish_kafka->setProperty(processors::PublishKafka::ClientName, "test_client"); - publish_kafka->setProperty(processors::PublishKafka::SeedBrokers, "test_seedbroker"); - publish_kafka->setProperty(processors::PublishKafka::QueueBufferMaxMessage, "1000"); - publish_kafka->setProperty(processors::PublishKafka::BatchSize, "1500"); + publish_kafka->setProperty(processors::PublishKafka::ClientName.name, "test_client"); + publish_kafka->setProperty(processors::PublishKafka::SeedBrokers.name, "test_seedbroker"); + publish_kafka->setProperty(processors::PublishKafka::QueueBufferMaxMessage.name, "1000"); + publish_kafka->setProperty(processors::PublishKafka::BatchSize.name, "1500"); REQUIRE_THROWS_WITH(test_controller.trigger(""), "Process Schedule Operation: Invalid configuration: Batch Size cannot be larger than Queue Max Message"); } TEST_CASE("Compress Codec property") { using processors::PublishKafka; SingleProcessorTestController test_controller(std::make_unique("PublishKafka")); - test_controller.getProcessor()->setProperty(PublishKafka::ClientName, "test_client"); - test_controller.getProcessor()->setProperty(PublishKafka::SeedBrokers, "test_seedbroker"); - test_controller.getProcessor()->setProperty(PublishKafka::Topic, "test_topic"); - test_controller.getProcessor()->setProperty(PublishKafka::MessageTimeOut, "10ms"); + test_controller.getProcessor()->setProperty(PublishKafka::ClientName.name, "test_client"); + test_controller.getProcessor()->setProperty(PublishKafka::SeedBrokers.name, "test_seedbroker"); + test_controller.getProcessor()->setProperty(PublishKafka::Topic.name, "test_topic"); + test_controller.getProcessor()->setProperty(PublishKafka::MessageTimeOut.name, "10ms"); SECTION("none") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "none")); + REQUIRE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "none")); REQUIRE_NOTHROW(test_controller.trigger("input")); } SECTION("gzip") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "gzip")); + REQUIRE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "gzip")); REQUIRE_NOTHROW(test_controller.trigger("input")); } SECTION("snappy") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "snappy")); + REQUIRE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "snappy")); REQUIRE_NOTHROW(test_controller.trigger("input")); } SECTION("lz4") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "lz4")); + REQUIRE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "lz4")); REQUIRE_NOTHROW(test_controller.trigger("input")); } SECTION("zstd") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "zstd")); + REQUIRE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "zstd")); REQUIRE_NOTHROW(test_controller.trigger("input")); } SECTION("foo") { - REQUIRE_NOTHROW(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec, "foo")); - REQUIRE_THROWS(test_controller.trigger("input")); + REQUIRE_FALSE(test_controller.getProcessor()->setProperty(PublishKafka::CompressCodec.name, "foo")); } } diff --git a/extensions/kubernetes/controllerservice/KubernetesControllerService.cpp b/extensions/kubernetes/controllerservice/KubernetesControllerService.cpp index e114457e40..be9f1618f9 100644 --- a/extensions/kubernetes/controllerservice/KubernetesControllerService.cpp +++ b/extensions/kubernetes/controllerservice/KubernetesControllerService.cpp @@ -58,19 +58,16 @@ void KubernetesControllerService::onEnable() { logger_->log_error("Could not create the API client in the Kubernetes Controller Service: {}", ex.what()); } - std::string namespace_filter; - if (getProperty(NamespaceFilter, namespace_filter) && !namespace_filter.empty()) { - namespace_filter_ = utils::Regex{namespace_filter}; + if (std::string namespace_filter = getProperty(NamespaceFilter.name).value_or(""); !namespace_filter.empty()) { + namespace_filter_ = utils::Regex{std::move(namespace_filter)}; } - std::string pod_name_filter; - if (getProperty(PodNameFilter, pod_name_filter) && !pod_name_filter.empty()) { - pod_name_filter_ = utils::Regex{pod_name_filter}; + if (std::string pod_name_filter = getProperty(PodNameFilter.name).value_or(""); !pod_name_filter.empty()) { + pod_name_filter_ = utils::Regex{std::move(pod_name_filter)}; } - std::string container_name_filter; - if (getProperty(ContainerNameFilter, container_name_filter) && !container_name_filter.empty()) { - container_name_filter_ = utils::Regex{container_name_filter}; + if (std::string container_name_filter = getProperty(ContainerNameFilter.name).value_or(""); !container_name_filter.empty()) { + container_name_filter_ = utils::Regex{std::move(container_name_filter)}; } } diff --git a/extensions/libarchive/BinFiles.cpp b/extensions/libarchive/BinFiles.cpp index 0d0e109849..7c3c04433a 100644 --- a/extensions/libarchive/BinFiles.cpp +++ b/extensions/libarchive/BinFiles.cpp @@ -15,19 +15,22 @@ * limitations under the License. */ #include "BinFiles.h" + #include +#include +#include #include -#include -#include #include +#include #include -#include -#include #include -#include "utils/StringUtils.h" +#include + #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" +#include "utils/StringUtils.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::processors { @@ -48,36 +51,33 @@ void BinFiles::initialize() { } void BinFiles::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { - uint32_t val32 = 0; - uint64_t val64 = 0; - if (context.getProperty(MinSize, val64)) { - this->binManager_.setMinSize(val64); + if (const auto val64 = utils::parseOptionalU64Property(context, MinSize)) { + this->binManager_.setMinSize(*val64); logger_->log_debug("BinFiles: MinSize [{}]", val64); } - if (context.getProperty(MaxSize, val64)) { - this->binManager_.setMaxSize(val64); + if (const auto val64 = utils::parseOptionalU64Property(context, MaxSize)) { + this->binManager_.setMaxSize(*val64); logger_->log_debug("BinFiles: MaxSize [{}]", val64); } - if (context.getProperty(MinEntries, val32)) { - this->binManager_.setMinEntries(val32); - logger_->log_debug("BinFiles: MinEntries [{}]", val32); + if (const auto val64 = utils::parseOptionalU64Property(context, MinEntries)) { + this->binManager_.setMinEntries(gsl::narrow(*val64)); + logger_->log_debug("BinFiles: MinEntries [{}]", *val64); } - if (context.getProperty(MaxEntries, val32)) { - this->binManager_.setMaxEntries(val32); - logger_->log_debug("BinFiles: MaxEntries [{}]", val32); + if (const auto val64 = utils::parseOptionalU64Property(context, MaxEntries)) { + this->binManager_.setMaxEntries(gsl::narrow(*val64)); + logger_->log_debug("BinFiles: MaxEntries [{}]", *val64); } - if (context.getProperty(MaxBinCount, maxBinCount_)) { - logger_->log_debug("BinFiles: MaxBinCount [{}]", maxBinCount_); - } - if (auto max_bin_age = context.getProperty(MaxBinAge)) { + maxBinCount_ = gsl::narrow(utils::parseU64Property(context, MaxBinCount)); + logger_->log_debug("BinFiles: MaxBinCount [{}]", maxBinCount_); + + if (auto max_bin_age = utils::parseOptionalMsProperty(context, MaxBinAge)) { // We need to trigger the processor even when there are no incoming flow files so that it can flush the bins. setTriggerWhenEmpty(true); - this->binManager_.setBinAge(max_bin_age->getMilliseconds()); - logger_->log_debug("BinFiles: MaxBinAge [{}]", max_bin_age->getMilliseconds()); - } - if (context.getProperty(BatchSize, batchSize_)) { - logger_->log_debug("BinFiles: BatchSize [{}]", batchSize_); + this->binManager_.setBinAge(*max_bin_age); + logger_->log_debug("BinFiles: MaxBinAge [{}]", *max_bin_age); } + batchSize_ = gsl::narrow(utils::parseU64Property(context, BatchSize)); + logger_->log_debug("BinFiles: BatchSize [{}]", batchSize_); } void BinFiles::preprocessFlowFile(const std::shared_ptr& flow) { diff --git a/extensions/libarchive/BinFiles.h b/extensions/libarchive/BinFiles.h index e90b708c09..6f53e6068e 100644 --- a/extensions/libarchive/BinFiles.h +++ b/extensions/libarchive/BinFiles.h @@ -189,34 +189,34 @@ class BinFiles : public core::ProcessorImpl { EXTENSIONAPI static constexpr auto MinSize = core::PropertyDefinitionBuilder<>::createProperty("Minimum Group Size") .withDescription("The minimum size of for the bundle") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("0") .build(); EXTENSIONAPI static constexpr auto MaxSize = core::PropertyDefinitionBuilder<>::createProperty("Maximum Group Size") .withDescription("The maximum size for the bundle. If not specified, there is no maximum.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_LONG_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto MinEntries = core::PropertyDefinitionBuilder<>::createProperty("Minimum Number of Entries") .withDescription("The minimum number of files to include in a bundle") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_INT_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .withDefaultValue("1") .build(); EXTENSIONAPI static constexpr auto MaxEntries = core::PropertyDefinitionBuilder<>::createProperty("Maximum Number of Entries") .withDescription("The maximum number of files to include in a bundle. If not specified, there is no maximum.") - .withPropertyType(core::StandardPropertyTypes::UNSIGNED_INT_TYPE) + .withValidator(core::StandardPropertyTypes::UNSIGNED_INTEGER_VALIDATOR) .build(); EXTENSIONAPI static constexpr auto MaxBinAge = core::PropertyDefinitionBuilder<>::createProperty("Max Bin Age") .withDescription("The maximum age of a Bin that will trigger a Bin to be complete. Expected format is