From 717be7e1bc7e31127aa692781bcf37faf449d5f5 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:58:57 -0700 Subject: [PATCH] various updates to initial repo setup --- .devcontainer/Dockerfile | 69 ++++++++ .devcontainer/devcontainer.json | 38 +++++ .devcontainer/docker-entrypoint.sh | 22 +++ .devcontainer/jpo-deduplicator.code-workspace | 11 ++ .devcontainer/kafka | 36 +++++ .devcontainer/post-create.sh | 30 ++++ .gitmodules | 2 +- .vscode/extensions.json | 8 + Makefile | 48 ++++++ README.md | 149 +++++++++++++++--- docker-compose-ode.yml | 8 +- docker-compose.yml | 14 +- sample.env | 71 ++------- 13 files changed, 413 insertions(+), 93 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-entrypoint.sh create mode 100644 .devcontainer/jpo-deduplicator.code-workspace create mode 100644 .devcontainer/kafka create mode 100644 .devcontainer/post-create.sh create mode 100644 .vscode/extensions.json create mode 100644 Makefile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..bad1a85 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,69 @@ +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- + +FROM eclipse-temurin:21-jdk + +# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser" +# property in devcontainer.json to use it. On Linux, the container user's GID/UIDs +# will be updated to match your local UID/GID (when using the dockerFile property). +# See https://aka.ms/vscode-remote/containers/non-root-user for details. +ARG USERNAME=vscode +ARG USER_UID=1234 +ARG USER_GID=$USER_UID + +# Configure apt +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update +RUN apt-get upgrade -y +RUN apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + +# Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user. +RUN groupadd --gid $USER_GID $USERNAME +RUN useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME +# [Optional] Add sudo support for the non-root user +RUN apt-get install -y sudo +RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME +RUN chmod 0440 /etc/sudoers.d/$USERNAME + +# Verify git, needed tools installed +RUN apt-get -y install git openssh-client less iproute2 procps curl lsb-release zip unzip sed kafkacat telnet + +#-------------------Install SDKMan---------------------------------- +RUN curl -s https://get.sdkman.io | bash +RUN chmod a+x "$HOME/.sdkman/bin/sdkman-init.sh" +#------------------------------------------------------------------------------------------------------------- + +#-------------------Install Maven CLI Tools---------------------------------- +ARG MAVEN_VERSION=3.6.3 +RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && sdk install maven ${MAVEN_VERSION}" +#------------------------------------------------------------------------------------------------------------- + +# Install snmp (rsu-data-controller dependency) +RUN apt-get -y install snmp + +#-------------------Install Kafka---------------------------------- +RUN mkdir ~/Downloads +RUN curl "https://archive.apache.org/dist/kafka/3.6.1/kafka_2.12-3.6.1.tgz" -o ~/Downloads/kafka.tgz +RUN mkdir ~/kafka \ + && cd ~/kafka \ + && tar -xvzf ~/Downloads/kafka.tgz --strip 1 +RUN echo "\nadvertised.listeners=PLAINTEXT://localhost:9092" >> ~/kafka/config/server.properties +#------------------------------------------------------------------------------------------------------------- + +# Clean up +RUN apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +ENV DEBIAN_FRONTEND=dialog + +# Allow for a consistant java home location for settings - image is changing over time +RUN if [ ! -d "/docker-java-home" ]; then ln -s "${JAVA_HOME}" /docker-java-home; fi + +COPY kafka /etc/init.d/kafka +RUN chmod 775 /etc/init.d/kafka && update-rc.d kafka defaults + +COPY ./docker-entrypoint.sh ./docker-entrypoint.sh +RUN chmod +x ./docker-entrypoint.sh +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..d808b0e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.154.0/containers/java-8 +{ + "name": "JPO Deduplicator Java 21", + "dockerFile": "Dockerfile", + "overrideCommand": false, + "shutdownAction": "stopContainer", + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "vscjava.vscode-java-pack", + "vscjava.vscode-java-debug", + "vscjava.vscode-maven", + "vscjava.vscode-java-dependency", + "vscjava.vscode-java-test", + "hbenl.vscode-test-explorer", + "ms-vscode.test-adapter-converter", + "esbenp.prettier-vscode", + "mhutchie.git-graph", + "tabnine.tabnine-vscode", + "redhat.java", + "redhat.vscode-commons" + ] + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [8080, 9090, 46753, 46800, 5555, 6666, 8090, 2181, 9092], + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "bash .devcontainer/post-create.sh", + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" + "runArgs": [ + "--network=host" + ] +} \ No newline at end of file diff --git a/.devcontainer/docker-entrypoint.sh b/.devcontainer/docker-entrypoint.sh new file mode 100644 index 0000000..a294d8b --- /dev/null +++ b/.devcontainer/docker-entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +trap stop SIGTERM SIGINT SIGQUIT SIGHUP ERR + +start() { + echo "Container started, starting kafka..." >>/root/startup.txt + sudo service kafka start +} + +stop() { + echo "Container stopped, performing cleanup..." >>/root/shutdown.txt + sudo service kafka stop + sleep 2 + exit 0 +} + +start + +while + sleep 1 & + wait $! +do :; done diff --git a/.devcontainer/jpo-deduplicator.code-workspace b/.devcontainer/jpo-deduplicator.code-workspace new file mode 100644 index 0000000..998bc22 --- /dev/null +++ b/.devcontainer/jpo-deduplicator.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": { + "java.configuration.updateBuildConfiguration": "automatic", + "java.server.launchMode": "Standard" + } +} \ No newline at end of file diff --git a/.devcontainer/kafka b/.devcontainer/kafka new file mode 100644 index 0000000..be2be98 --- /dev/null +++ b/.devcontainer/kafka @@ -0,0 +1,36 @@ +DAEMON_PATH=/root/kafka +PATH=$PATH:$DAEMON_PATH/bin +PATH=$PATH:/root/.sdkman/candidates/java/current/bin + +export LOG_DIR=/var/log/kafka + +# See how we were called. +case "$1" in + start) + # Start daemon. + echo "Starting Kafka"; + $DAEMON_PATH/bin/kafka-server-start.sh -daemon $DAEMON_PATH/config/server.properties + ;; + stop) + # Stop daemons. + echo "Shutting down Kafka"; + $DAEMON_PATH/bin/kafka-server-stop.sh + ;; + restart) + $0 stop + sleep 2 + $0 start + ;; + status) + pid=`ps ax | grep -i 'kafka.Kafka' | grep -v grep | awk '{print $1}'` + if [ -n "$pid" ] + then + echo "Kafka is Running as PID: $pid" + else + echo "Kafka is not Running" + fi + ;; + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 +esac \ No newline at end of file diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100644 index 0000000..b54a6a5 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,30 @@ +cd ~/kafka/ +KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + +bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c config/kraft/server.properties + +# start kafka +bin/kafka-server-start.sh -daemon config/kraft/server.properties +# wait 2 seconds for the server to start and be able to add partitions +sleep 2s +# add topics + +# MAP +bin/kafka-topics.sh --create --topic "topic.OdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.ProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 + +# SPaT +bin/kafka-topics.sh --create --topic "topic.OdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.ProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 + +# BSM +bin/kafka-topics.sh --create --topic "topic.OdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 + +# TIM +bin/kafka-topics.sh --create --topic "topic.OdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 +bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 diff --git a/.gitmodules b/.gitmodules index eda54ba..532f96c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "jpo-utils"] path = jpo-utils - url = git@github.com:usdot-jpo-ode/jpo-utils.git + url = https://github.com/usdot-jpo-ode/jpo-utils.git diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..53f13e6 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "redhat.vscode-xml", + "vscjava.vscode-java-pack", + "redhat.java", + "vmware.vscode-boot-dev-pack" + ] +} \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d080e5c --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +default: + $(info Make target options:) + $(info `make start` to run the deduplicator) + $(info `make build` to build the deduplicator) + $(info `make stop` to stop the deduplicator) + $(info `make delete` to stop the deduplicator and remove the volumes) + $(info `make restart` to stop and then start the deduplicator) + $(info `make rebuild` to stop, delete, and then rebuild the containers) + $(info `make clean-build` to rebuild the containers without using the cache) + +.PHONY: start +start: +ifeq ("$(wildcard .env)", "") + $(error "ERROR: deduplicator Environment file `.env` not found in ${PWD}") +endif +ifeq ("$(wildcard ./jpo-utils/.env)", "") + $(error "ERROR: jpo-utils Environment file `.env` not found in ${PWD}") +endif + docker compose up -d + +build: +ifeq ("$(wildcard .env)", "") + $(error "ERROR: jpo-ode Environment file `.env` not found in ${PWD}") +endif +ifeq ("$(wildcard ./jpo-utils/.env)", "") + $(error "ERROR: jpo-utils Environment file `.env` not found in ${PWD}") +endif + docker compose build + +.PHONY: stop +stop: + docker compose down + +.PHONY: delete +delete: + docker compose down -v + +.PHONY: restart +restart: + $(MAKE) stop start + +.PHONY: rebuild +rebuild: + $(MAKE) stop delete build start + +.PHONY: clean-build +clean-build: + docker compose build --no-cache \ No newline at end of file diff --git a/README.md b/README.md index 5d0de59..983dec0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # jpo-deduplicator - +**US Department of Transportation (USDOT) Intelligent Transportation Systems (ITS) Joint Program Office (JPO) Message Deduplicator** -## jpo-deduplicator -The JPO-Deduplicator is a Kafka Java spring-boot application designed to reduce the number of messages stored and processed in the ODE system. This is done by reading in messages from an input topic (such as topic.ProcessedMap) and outputting a subset of those messages on a related output topic (topic.DeduplicatedProcessedMap). Functionally, this is done by removing deduplicate messages from the input topic and only passing on unique messages. In addition, each topic will pass on at least 1 message per hour even if the message is a duplicate. This behavior helps ensure messages are still flowing through the system. The following topics currently support deduplication. +The JPO-Deduplicator is a Kafka Java spring-boot application designed to reduce the number of messages stored and processed in the ODE system. This is done by reading in messages from an input topic (such as topic.ProcessedMap) and outputting a subset of those messages on a related output topic (topic.DeduplicatedProcessedMap). Functionally, this is done by removing deduplicate messages from the input topic and only passing on unique messages. In addition, each topic will pass on at least 1 message per hour even if the message is a duplicate. This behavior helps ensure messages are still flowing through the system. + +The following topics currently support deduplication. - topic.ProcessedMap -> topic.DeduplicatedProcessedMap - topic.ProcessedMapWKT -> topic.DeduplicatedProcessedMapWKT @@ -14,9 +15,87 @@ The JPO-Deduplicator is a Kafka Java spring-boot application designed to reduce - topic.OdeBsmJson -> topic.DeduplicatedOdeBsmJson - topic.ProcessedSpat -> topic.DeduplicatedProcessedSpat -### Deduplication Config +## Release Notes + +The current version and release history of the JPO Deduplicator: [Release Notes]() + + + +## Table of Contents + +- [jpo-deduplicator](#jpo-deduplicator) + - [Release Notes](#release-notes) + - [Table of Contents](#table-of-contents) + - [1. System Requirements](#1-system-requirements) + - [2. Usage Example](#2-usage-example) + - [3. Configuration](#3-configuration) + - [Software Prerequisites](#software-prerequisites) + - [Feature Flags](#feature-flags) + - [Generate a Github Token](#generate-a-github-token) + - [4. Development Setup](#4-development-setup) + - [Integrated Development Environment (IDE)](#integrated-development-environment-ide) + - [5. Contact Information](#5-contact-information) + - [License information](#license-information) + - [6. Contributing](#6-contributing) + + + +## 1. System Requirements + +Recommended machine specs running Docker to run the JPO-Deduplicator: + +- Minimum RAM: 16 GB + +- Minimum storage space: 100 GB +- Supported operating systems: + + - Ubuntu 20.04 Linux (Recommended) + - Windows 10/11 Professional (Professional version required for Docker virtualization) + - OSX 10 Mojave + +The JPO-Deduplicator software can run on most standard Window, Mac, or Linux based computers with +Pentium core processors. Performance of the software will be based on the computing power and available RAM in +the system. Larger data flows can require much larger space requirements depending on the amount of data being processed by the software. The JPO-Deduplicator software application was developed using the open source programming language Java. If running the JPO-Deduplicator outside of Docker, the application requires the Java 21 runtime environment. + +[Back to top](#table-of-contents) + + + +## 2. Usage Example + +1. Create a copy of `sample.env` and rename it to `.env`. +2. Create a copy of the `jpo-utils/sample.env` file and rename it to `.env` in the `jpo-utils` directory. +3. Update the variable `MAVEN_GITHUB_TOKEN` in the root `.env` file to a github token used for downloading jar file dependencies. For full instructions on how to generate a token please see [here](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages). +4. Navigate back to the root directory and run the following command: `make build` + 1. Make sure that you have [docker-compose](https://docs.docker.com/compose/install/) and [make](https://www.gnu.org/software/make/manual/make.html) installed. +5. Run the following command: `make start` +6. Produce a sample message to one of the sink topics by using `kafka_ui` by: + 1. Go to `localhost:8001` + 2. Click local -> Topics + 3. Select `topic.OdeMapJson` + 4. Select `Produce Message` + 5. Copy in sample JSON for a Map Message + 6. Click `Produce Message` multiple times +7. View the synced message in `kafka_ui` by: + 1. Go to `localhost:8001` + 2. Click local -> Topics + 3. Select `topic.DeduplicatedOdeMapJson` + 4. You should now see only one copy of the map message sent. + +[Back to top](#table-of-contents) + + + +## 3. Configuration + +### Software Prerequisites -When running the jpo-deduplication as a submodule in jpo-utils, the deduplicator will automatically turn on deduplication for a topic when that topic is created. For example if the KAFKA_TOPIC_CREATE_GEOJSONCONVERTER environment variable is set to true, the deduplicator will start performing deduplication for ProcessedMap, ProcessedMapWKT, and ProcessedSpat data. +The JPO-Deduplicator is a micro service that runs as an independent application but serves the sole purpose of deduplicating JSON objects created by the JPO-ODE via Apache Kafka. To support these JSON objects, the JPO-Deduplicator application utilizes some classes from the JPO-ODE, JPO-GeojsonConverter, and the JPO-ConflictMonitor. These classes are referenced in the JPO-Deduplicator by pulling the built `.jar` artifact from GitHub Maven Central. All other required dependencies will automatically be downloaded and installed as part of the Docker build process. + +- Docker: +- Docker-Compose: + +### Feature Flags To manually configure deduplication for a topic, the following environment variables can also be used. @@ -47,21 +126,45 @@ For local development the following steps are also required 8. Create a copy of [settings.xml](jpo-deduplicator/jpo-deduplicator/settings.xml) and save it to `~/.m2/settings.xml` 9. Update the variables in your `~/.m2/settings.xml` with the token value and target jpo-ode organization. -### Quick Run -1. Create a copy of `sample.env` and rename it to `.env`. -2. Update the variable `MAVEN_GITHUB_TOKEN` to a github token used for downloading jar file dependencies. For full instructions on how to generate a token please see here: -3. Set the password for `MONGO_ADMIN_DB_PASS` and `MONGO_READ_WRITE_PASS` environmental variables to a secure password. -4. Set the `COMPOSE_PROFILES` variable to: `kafka,kafka_ui,kafka_setup, jpo-deduplicator` -5. Navigate back to the root directory and run the following command: `docker compose up -d` -6. Produce a sample message to one of the sink topics by using `kafka_ui` by: - 1. Go to `localhost:8001` - 2. Click local -> Topics - 3. Select `topic.OdeMapJson` - 4. Select `Produce Message` - 5. Copy in sample JSON for a Map Message - 6. Click `Produce Message` multiple times -7. View the synced message in `kafka_ui` by: - 1. Go to `localhost:8001` - 2. Click local -> Topics - 3. Select `topic.DeduplicatedOdeMapJson` - 4. You should now see only one copy of the map message sent. \ No newline at end of file +[Back to top](#table-of-contents) + + + +## 4. Development Setup + +### Integrated Development Environment (IDE) + +Install the IDE of your choice: + +- Eclipse: [https://eclipse.org/](https://eclipse.org/) +- STS: [https://spring.io/tools/sts/all](https://spring.io/tools/sts/all) +- IntelliJ: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/) +- VS Code: [https://code.visualstudio.com/](https://code.visualstudio.com/) + +[Back to top](#table-of-contents) + + + +## 5. Contact Information + +Contact the developers of the JPO-Deduplicator application by submitting a [Github issue](https://github.com/usdot-jpo-ode/jpo-deduplicator/issues). + +### License information + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +file except in compliance with the License. + +You may obtain a copy of the License at +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the License for the specific language governing +permissions and limitations under the [License](http://www.apache.org/licenses/LICENSE-2.0). + +[Back to top](#table-of-contents) + + + +## 6. Contributing + +Please read the ODE [contributing guide](https://github.com/usdot-jpo-ode/jpo-ode/blob/develop/docs/contributing_guide.md) to learn about our development process, how to propose pull requests and improvements, and how to build and test your changes to this project. + +[Back to top](#table-of-contents) \ No newline at end of file diff --git a/docker-compose-ode.yml b/docker-compose-ode.yml index bdcdc8c..c96d43f 100644 --- a/docker-compose-ode.yml +++ b/docker-compose-ode.yml @@ -5,7 +5,7 @@ services: geojsonconverter: profiles: - all - - cm_api_full + - deduplicator_full - geojsonconverter image: usdotjpoode/geojsonconverter:latest restart: ${RESTART_POLICY} @@ -33,7 +33,7 @@ services: ode: profiles: - all - - cm_api_full + - deduplicator_full - ode image: usdotjpoode/jpo-ode:latest restart: ${RESTART_POLICY} @@ -87,7 +87,7 @@ services: adm: profiles: - all - - cm_api_full + - deduplicator_full - adm image: usdotjpoode/asn1_codec:latest restart: ${RESTART_POLICY} @@ -117,7 +117,7 @@ services: conflictmonitor: profiles: - all - - cm_api_full + - deduplicator_full - conflictmonitor image: usdotjpoode/jpo-conflictmonitor:latest restart: ${RESTART_POLICY} diff --git a/docker-compose.yml b/docker-compose.yml index 142814a..a81f59d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,8 @@ services: deduplicator: profiles: - all - - deduplicator + - deduplicator_base + - deduplicator_full build: context: . dockerfile: Dockerfile @@ -24,10 +25,9 @@ services: ENABLE_ODE_RAW_ENCODED_TIM_DEDUPLICATION: ${ENABLE_ODE_RAW_ENCODED_TIM_DEDUPLICATION} ENABLE_PROCESSED_SPAT_DEDUPLICATION: ${ENABLE_PROCESSED_SPAT_DEDUPLICATION} ENABLE_ODE_BSM_DEDUPLICATION: ${ENABLE_ODE_BSM_DEDUPLICATION} - KAFKA_TYPE: CONFLUENT - CONFLUENT_KEY: 12345 - CONFLUENT_SECRET: 12345 - + KAFKA_TYPE: ${KAFKA_TYPE} + CONFLUENT_KEY: ${CONFLUENT_KEY} + CONFLUENT_SECRET: ${CONFLUENT_SECRET} healthcheck: test: ["CMD", "java", "-version"] interval: 10s @@ -41,3 +41,7 @@ services: resources: limits: memory: 3G + depends_on: + kafka: + condition: service_healthy + required: false \ No newline at end of file diff --git a/sample.env b/sample.env index fa66e3c..cf0a681 100644 --- a/sample.env +++ b/sample.env @@ -5,14 +5,10 @@ # Set the HOST IP of where the containers are running DOCKER_HOST_IP= - -# Set to the directory where the source code is. Generally the location of this file. -DOCKER_HOST_DIR= - - KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092 # GitHub properties for pulling the latest version of the JPO-ODE +# Required for building the deduplicator. Documentation: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages MAVEN_GITHUB_TOKEN= MAVEN_GITHUB_ORG=usdot-jpo-ode @@ -22,33 +18,17 @@ RESTART_POLICY="on-failure:3" # Available profiles: # - all -# - deduplicator +# - deduplicator_base +# - deduplicator +# - deduplicator_full # - deduplicator - -# Available profiles: -# - all -# - cm_api_full -# - conflictmonitor -# - geojsonconverter -# - cm_gui -# - cm_api -# - keycloak -# - mongo -# - ode -# - adm -# - kafka -# - cm_base -# - conflictmonitor-build -# - cm_build -# - conflictmonitor-build -# - cm_release -# - conflictmonitor-release -# - ode_geojsonconverter # - geojsonconverter # - ode # - adm - -COMPOSE_PROFILES=deduplicator +# +# All profiles from the jpo-utils repo are also allowed +# Example: COMPOSE_PROFILES=kafka,kafka_ui,kafka_setup, deduplicator_full +COMPOSE_PROFILES=kafka,kafka_ui,kafka_setup, deduplicator_full # Set to "CONFLUENT" if broker is a Confluent Cloud broker @@ -58,42 +38,13 @@ KAFKA_TYPE= CONFLUENT_KEY= CONFLUENT_SECRET= +KAFKA_TOPIC_CREATE_DEDUPLICATOR=true -### COMMON variables - END ### - - - -### KAFKA variables - START ### -KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092 -KAFKA_LOG_RETENTION_HOURS=3 -KAFKA_LOG_RETENTION_BYTES=10737418240 # 10GB - -# Variables for creating kafka topics: -KAFKA_TOPIC_PARTITIONS=1 -KAFKA_TOPIC_REPLICAS=1 -KAFKA_TOPIC_MIN_INSYNC_REPLICAS=1 -KAFKA_TOPIC_RETENTION_MS=300000 -KAFKA_TOPIC_DELETE_RETENTION_MS=3600000 -### KAFKA variables - END ### - - -### DEDUPLICATOR variables - START ### - -# Required for building the deduplicator. Documentation: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages -MAVEN_GITHUB_TOKEN= -MAVEN_GITHUB_ORG=usdot-jpo-ode - +# Deduplicator Feature Flags ENABLE_PROCESSED_MAP_DEDUPLICATION=true ENABLE_PROCESSED_MAP_WKT_DEDUPLICATION=true ENABLE_ODE_MAP_DEDUPLICATION=true ENABLE_ODE_TIM_DEDUPLICATION=true ENABLE_ODE_RAW_ENCODED_TIM_DEDUPLICATION=true ENABLE_PROCESSED_SPAT_DEDUPLICATION=true -ENABLE_ODE_BSM_DEDUPLICATION=true - - - -### DEDUPLICATOR variables - END ### - - - +ENABLE_ODE_BSM_DEDUPLICATION=true \ No newline at end of file