From 9f40ef423778a18405a6cf3fe5ecc5b42308613b Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:14:02 -0700 Subject: [PATCH 1/2] change the max tasks to be based off of the CONNECT_TASKS_MAX env var --- jikkou/kafka-connectors-template.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jikkou/kafka-connectors-template.jinja b/jikkou/kafka-connectors-template.jinja index bb97cb0..d3fc090 100644 --- a/jikkou/kafka-connectors-template.jinja +++ b/jikkou/kafka-connectors-template.jinja @@ -12,7 +12,7 @@ metadata: kafka.jikkou.io/connect-cluster: {{ values.clusterName }} spec: connectorClass: {{ system.env.KAFKA_CONNECT_CONNECTOR_CLASS | default(values.connectorClass) }} - tasksMax: {{ system.env.KAFKA_TOPIC_PARTITIONS | default(values.tasksMax) }} + tasksMax: {{ system.env.CONNECT_TASKS_MAX | default(values.tasksMax) }} config: collection: {{ connector.collectionName }} connection.uri: mongodb://{{ system.env.MONGO_CONNECTOR_USERNAME }}:{{ system.env.MONGO_CONNECTOR_PASSWORD }}@{{ system.env.MONGO_DB_IP }}:27017/ From a02104d452bd638df40929daac6be531fb47e470 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:13:37 -0700 Subject: [PATCH 2/2] updates for deduplicated processed BSM --- jikkou/kafka-connectors-template.jinja | 6 +----- jikkou/kafka-connectors-values.yaml | 13 +++++++------ jikkou/kafka-topics-values.yaml | 3 ++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/jikkou/kafka-connectors-template.jinja b/jikkou/kafka-connectors-template.jinja index d3fc090..f4f0adf 100644 --- a/jikkou/kafka-connectors-template.jinja +++ b/jikkou/kafka-connectors-template.jinja @@ -12,7 +12,7 @@ metadata: kafka.jikkou.io/connect-cluster: {{ values.clusterName }} spec: connectorClass: {{ system.env.KAFKA_CONNECT_CONNECTOR_CLASS | default(values.connectorClass) }} - tasksMax: {{ system.env.CONNECT_TASKS_MAX | default(values.tasksMax) }} + tasksMax: {{ system.env.KAFKA_TOPIC_PARTITIONS | default(values.tasksMax) }} config: collection: {{ connector.collectionName }} connection.uri: mongodb://{{ system.env.MONGO_CONNECTOR_USERNAME }}:{{ system.env.MONGO_CONNECTOR_PASSWORD }}@{{ system.env.MONGO_DB_IP }}:27017/ @@ -56,10 +56,6 @@ spec: {{ create_connector(values.apps.ode) }} {% endif %} -{% if system.env.CONNECT_CREATE_GEOJSONCONVERTER %} -{{ create_connector(values.apps.geojsonconverter) }} -{% endif %} - {% if system.env.CONNECT_CREATE_CONFLICTMONITOR %} {{ create_connector(values.apps.conflictmonitor) }} {% endif %} diff --git a/jikkou/kafka-connectors-values.yaml b/jikkou/kafka-connectors-values.yaml index 8034c4b..f45b2b1 100644 --- a/jikkou/kafka-connectors-values.yaml +++ b/jikkou/kafka-connectors-values.yaml @@ -103,12 +103,6 @@ apps: - topicName: topic.OdeRawEncodedTIMJson collectionName: OdeRawEncodedTIMJson generateTimestamp: true - geojsonconverter: - name: geojsonconverter - connectors: - - topicName: topic.ProcessedBsm - collectionName: ProcessedBsm - generateTimestamp: true geojsonconverter_duplicated: name: geojsonconverter-duplicated connectors: @@ -118,6 +112,9 @@ apps: - topicName: topic.ProcessedSpat collectionName: ProcessedSpat generateTimestamp: true + - topicName: topic.ProcessedBsm + collectionName: ProcessedBsm + generateTimestamp: true deduplicator: name: deduplicator connectors: @@ -145,6 +142,10 @@ apps: collectionName: ProcessedSpat generateTimestamp: true connectorName: DeduplicatedProcessedSpat + - topicName: topic.DeduplicatedProcessedBsm + collectionName: ProcessedBsm + generateTimestamp: true + connectorName: DeduplicatedProcessedBsm conflictmonitor: name: conflictmonitor connectors: diff --git a/jikkou/kafka-topics-values.yaml b/jikkou/kafka-topics-values.yaml index 3a4cc48..720460b 100644 --- a/jikkou/kafka-topics-values.yaml +++ b/jikkou/kafka-topics-values.yaml @@ -166,7 +166,8 @@ apps: - topic.DeduplicatedOdeMapJson - topic.DeduplicatedOdeTimJson - topic.DeduplicatedOdeRawEncodedTIMJson - - topic.DeduplicatedOdeBsmJson + - topic.DeduplicatedOdeBsmJson + - topic.DeduplicatedProcessedBsm tableTopics: {} customTopics: {} kafkaconnect: