From e9b60793af711bfd8ad53ad70d7327947fdee225 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:04:00 -0700 Subject: [PATCH 1/8] Combine GitHub Maven repositories to 1 wildcard repository to help with build times. --- .../jpo-deduplicator/settings.xml | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/jpo-deduplicator/jpo-deduplicator/settings.xml b/jpo-deduplicator/jpo-deduplicator/settings.xml index eaee228..2a4f90b 100644 --- a/jpo-deduplicator/jpo-deduplicator/settings.xml +++ b/jpo-deduplicator/jpo-deduplicator/settings.xml @@ -9,16 +9,6 @@ jpo_deduplicator ${env.MAVEN_GITHUB_TOKEN} - - github_jpo_ode - jpo_deduplicator - ${env.MAVEN_GITHUB_TOKEN} - - - github_jpo_geojsonconverter - jpo_deduplicator - ${env.MAVEN_GITHUB_TOKEN} - @@ -27,23 +17,7 @@ github GitHub JPO Conflict Monitor - https://maven.pkg.github.com/${env.MAVEN_GITHUB_ORG}/jpo-conflictmonitor - - false - - - - github_jpo_ode - GitHub JPO ODE - https://maven.pkg.github.com/${env.MAVEN_GITHUB_ORG}/jpo-ode - - false - - - - github_jpo_geojsonconverter - GitHub JPO GeojsonConverter - https://maven.pkg.github.com/${env.MAVEN_GITHUB_ORG}/jpo-geojsonconverter + https://maven.pkg.github.com/${env.MAVEN_GITHUB_ORG}/* false From 730cf1d2f389e955c316602199d590c5509e2195 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:17:36 -0700 Subject: [PATCH 2/8] adding docker caching to github actions --- .github/workflows/docker.yml | 4 +++- .github/workflows/dockerhub.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fb020e7..2a76f66 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,4 +21,6 @@ jobs: MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }} MAVEN_GITHUB_ORG=${{ github.repository_owner }} secrets: | - MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }} \ No newline at end of file + MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 00e2704..0dea5d5 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -36,4 +36,6 @@ jobs: MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }} MAVEN_GITHUB_ORG=${{ github.repository_owner }} secrets: | - MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }} \ No newline at end of file + MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file From 96d72345a3ab859a4f3adf59cf39fa8db3e1b3b4 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Thu, 2 Jan 2025 09:59:08 -0700 Subject: [PATCH 3/8] adding missing cm topic --- jikkou/kafka-topics-values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/jikkou/kafka-topics-values.yaml b/jikkou/kafka-topics-values.yaml index 0ddb483..422d577 100644 --- a/jikkou/kafka-topics-values.yaml +++ b/jikkou/kafka-topics-values.yaml @@ -155,6 +155,7 @@ apps: - topic.CmSignalStateConflictNotificationAggregation - topic.CmSpatTimeChangeDetailsNotificationAggregation - topic.CmEventStateProgressionNotificationAggregation + - topic.CmEventStateProgressionNotification customTopics: {} deduplicator: name: jpo-deduplicator From 76699cf15269191b1ec72a6924cb41a759ea7640 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:11:49 -0700 Subject: [PATCH 4/8] sets default credentials in sample.env for required variables --- README.md | 16 +++++++++++++--- sample.env | 12 ++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ea2e9d4..4f03f12 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,11 @@ The JPO ITS utilities repository serves as a central location for deploying open - [Configuration](#configuration) - [Configure Kafka Connector Creation](#configure-kafka-connector-creation) - [Quick Run](#quick-run-2) - - [5. Deduplicator](#5-jpo-Deduplicator) - - [Deduplication Configuration](#deduplication-config) - - [Github Token Generation](#generate-a-github-token) + - [5. jpo-deduplicator](#5-jpo-deduplicator) + - [Deduplication Config](#deduplication-config) + - [Generate a Github Token](#generate-a-github-token) - [Quick Run](#quick-run-3) + - [Security Notice](#security-notice) @@ -256,3 +257,12 @@ For local development the following steps are also required 4. You should now see only one copy of the map message sent. [Back to top](#toc) + +## Security Notice + +While default passwords are provided for development convenience, it is **strongly recommended** to: + +1. Change all passwords before deploying to any environment +2. Never use default passwords in production +3. Use secure password generation and management practices +4. Consider using Docker secrets or environment management tools for production deployments diff --git a/sample.env b/sample.env index 797d2aa..4229a12 100644 --- a/sample.env +++ b/sample.env @@ -62,19 +62,19 @@ MONGO_DB_NAME=CV # Generate a random string for the MongoDB keyfile using the following command: # $ openssl rand -base64 32 -MONGO_DB_KEYFILE_STRING= +MONGO_DB_KEYFILE_STRING=replace_me MONGO_ADMIN_DB_USER=admin -MONGO_ADMIN_DB_PASS= +MONGO_ADMIN_DB_PASS=replace_me MONGO_READ_WRITE_USER=ode -MONGO_READ_WRITE_PASS= +MONGO_READ_WRITE_PASS=replace_me MONGO_READ_USER=user -MONGO_READ_PASS= +MONGO_READ_PASS=replace_me MONGO_EXPORTER_USERNAME=export -MONGO_EXPORTER_PASSWORD= +MONGO_EXPORTER_PASSWORD=replace_me MONGO_EXPRESS_USER=${MONGO_ADMIN_DB_USER} MONGO_EXPRESS_PASS=${MONGO_ADMIN_DB_PASS} @@ -126,7 +126,7 @@ CONNECT_CONFIG_RELATIVE_PATH="./jikkou/kafka-connectors-values.yaml" ### DEDUPLICATOR variables - START ### # Required for building the deduplicator. Documentation: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages -MAVEN_GITHUB_TOKEN= +MAVEN_GITHUB_TOKEN=replace_me MAVEN_GITHUB_ORG=usdot-jpo-ode ENABLE_PROCESSED_MAP_DEDUPLICATION=${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER} From bfc5f9d8ea78e70a2246583305ab9fd048f59a1e Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:16:37 -0700 Subject: [PATCH 5/8] updates to have the deduplicator dockerfile check if build arguments are null --- docker-compose-deduplicator.yml | 11 ++++------- jpo-deduplicator/Dockerfile | 13 ++----------- sample.env | 12 ++++++------ 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/docker-compose-deduplicator.yml b/docker-compose-deduplicator.yml index 5c315d7..c1e7410 100644 --- a/docker-compose-deduplicator.yml +++ b/docker-compose-deduplicator.yml @@ -7,14 +7,14 @@ services: context: jpo-deduplicator dockerfile: Dockerfile args: - MAVEN_GITHUB_TOKEN: ${MAVEN_GITHUB_TOKEN:?error} - MAVEN_GITHUB_ORG: ${MAVEN_GITHUB_ORG:?error} + MAVEN_GITHUB_TOKEN: ${MAVEN_GITHUB_TOKEN} + MAVEN_GITHUB_ORG: ${MAVEN_GITHUB_ORG} image: jpo-deduplicator:latest restart: ${RESTART_POLICY} environment: DOCKER_HOST_IP: ${DOCKER_HOST_IP} - KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:?error} - spring.kafka.bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:?error} + KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS} + spring.kafka.bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS} enableProcessedMapDeduplication: ${ENABLE_PROCESSED_MAP_DEDUPLICATION} enableProcessedMapWktDeduplication: ${ENABLE_PROCESSED_MAP_WKT_DEDUPLICATION} enableOdeMapDeduplication: ${ENABLE_ODE_MAP_DEDUPLICATION} @@ -22,9 +22,6 @@ services: enableOdeRawEncodedTimDeduplication: ${ENABLE_ODE_RAW_ENCODED_TIM_DEDUPLICATION} enableProcessedSpatDeduplication: ${ENABLE_PROCESSED_SPAT_DEDUPLICATION} enableOdeBsmDeduplication: ${ENABLE_ODE_BSM_DEDUPLICATION} - - - healthcheck: test: ["CMD", "java", "-version"] interval: 10s diff --git a/jpo-deduplicator/Dockerfile b/jpo-deduplicator/Dockerfile index 6228e45..2631f79 100644 --- a/jpo-deduplicator/Dockerfile +++ b/jpo-deduplicator/Dockerfile @@ -8,17 +8,8 @@ ARG MAVEN_GITHUB_ORG ENV MAVEN_GITHUB_TOKEN=$MAVEN_GITHUB_TOKEN ENV MAVEN_GITHUB_ORG=$MAVEN_GITHUB_ORG -# COPY ./jpo-conflictmonitor/pom.xml ./jpo-conflictmonitor/ -# COPY ./settings.xml ./jpo-conflictmonitor/ - -# # Copy and Build Conflict Monitor -# # Download dependencies alone to cache them first -# WORKDIR /home/jpo-conflictmonitor -# RUN mvn -s settings.xml dependency:resolve - -# # Copy the source code and build the conflict monitor -# COPY ./jpo-conflictmonitor/src ./src -# RUN mvn -s settings.xml install -DskipTests -Ppackage-jar +RUN test -n "$MAVEN_GITHUB_TOKEN" || (echo "Error: MAVEN_GITHUB_TOKEN cannot be empty" && exit 1) +RUN test -n "$MAVEN_GITHUB_ORG" || (echo "Error: MAVEN_GITHUB_ORG cannot be empty" && exit 1) # Copy and Build Deduplicator WORKDIR /home diff --git a/sample.env b/sample.env index 4229a12..797d2aa 100644 --- a/sample.env +++ b/sample.env @@ -62,19 +62,19 @@ MONGO_DB_NAME=CV # Generate a random string for the MongoDB keyfile using the following command: # $ openssl rand -base64 32 -MONGO_DB_KEYFILE_STRING=replace_me +MONGO_DB_KEYFILE_STRING= MONGO_ADMIN_DB_USER=admin -MONGO_ADMIN_DB_PASS=replace_me +MONGO_ADMIN_DB_PASS= MONGO_READ_WRITE_USER=ode -MONGO_READ_WRITE_PASS=replace_me +MONGO_READ_WRITE_PASS= MONGO_READ_USER=user -MONGO_READ_PASS=replace_me +MONGO_READ_PASS= MONGO_EXPORTER_USERNAME=export -MONGO_EXPORTER_PASSWORD=replace_me +MONGO_EXPORTER_PASSWORD= MONGO_EXPRESS_USER=${MONGO_ADMIN_DB_USER} MONGO_EXPRESS_PASS=${MONGO_ADMIN_DB_PASS} @@ -126,7 +126,7 @@ CONNECT_CONFIG_RELATIVE_PATH="./jikkou/kafka-connectors-values.yaml" ### DEDUPLICATOR variables - START ### # Required for building the deduplicator. Documentation: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry#authenticating-to-github-packages -MAVEN_GITHUB_TOKEN=replace_me +MAVEN_GITHUB_TOKEN= MAVEN_GITHUB_ORG=usdot-jpo-ode ENABLE_PROCESSED_MAP_DEDUPLICATION=${KAFKA_TOPIC_CREATE_GEOJSONCONVERTER} From 974f8d9e7f3763aad0752457260deddd4b5b7848 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:23:33 -0700 Subject: [PATCH 6/8] setting default mongo passwords in sample.env --- sample.env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sample.env b/sample.env index 797d2aa..a67d94f 100644 --- a/sample.env +++ b/sample.env @@ -62,19 +62,19 @@ MONGO_DB_NAME=CV # Generate a random string for the MongoDB keyfile using the following command: # $ openssl rand -base64 32 -MONGO_DB_KEYFILE_STRING= +MONGO_DB_KEYFILE_STRING=replace_me MONGO_ADMIN_DB_USER=admin -MONGO_ADMIN_DB_PASS= +MONGO_ADMIN_DB_PASS=replace_me MONGO_READ_WRITE_USER=ode -MONGO_READ_WRITE_PASS= +MONGO_READ_WRITE_PASS=replace_me MONGO_READ_USER=user -MONGO_READ_PASS= +MONGO_READ_PASS=replace_me MONGO_EXPORTER_USERNAME=export -MONGO_EXPORTER_PASSWORD= +MONGO_EXPORTER_PASSWORD=replace_me MONGO_EXPRESS_USER=${MONGO_ADMIN_DB_USER} MONGO_EXPRESS_PASS=${MONGO_ADMIN_DB_PASS} From 3f9cd00e5dc99f0fb727635308aca280b188604b Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:57:14 -0700 Subject: [PATCH 7/8] adding missing ode topics --- jikkou/kafka-topics-values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jikkou/kafka-topics-values.yaml b/jikkou/kafka-topics-values.yaml index 422d577..05825f7 100644 --- a/jikkou/kafka-topics-values.yaml +++ b/jikkou/kafka-topics-values.yaml @@ -82,6 +82,12 @@ apps: - topic.OdeRawEncodedPSMJson - topic.OdePsmTxPojo - topic.OdePsmJson + - topic.OdeTimRxJson + - topic.OdeDNMsgJson + - topic.OdeBsmDuringEventPojo + - topic.OdeBsmRxPojo + - topic.OdeBsmTxPojo + - topic.OdeTimBroadcastPojo tableTopics: {} customTopics: {} geojsonconverter: From 06248a43406e5fedeb50b475e29bf0bd5b59e282 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:50:58 -0700 Subject: [PATCH 8/8] remove disabled topics --- jikkou/kafka-topics-values.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jikkou/kafka-topics-values.yaml b/jikkou/kafka-topics-values.yaml index 05825f7..7c29e20 100644 --- a/jikkou/kafka-topics-values.yaml +++ b/jikkou/kafka-topics-values.yaml @@ -83,11 +83,6 @@ apps: - topic.OdePsmTxPojo - topic.OdePsmJson - topic.OdeTimRxJson - - topic.OdeDNMsgJson - - topic.OdeBsmDuringEventPojo - - topic.OdeBsmRxPojo - - topic.OdeBsmTxPojo - - topic.OdeTimBroadcastPojo tableTopics: {} customTopics: {} geojsonconverter: