various updates to initial repo setup

.devcontainer/Dockerfile

@@ -0,0 +1,69 @@
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+
+FROM eclipse-temurin:21-jdk
+
+# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser"
+# property in devcontainer.json to use it. On Linux, the container user's GID/UIDs
+# will be updated to match your local UID/GID (when using the dockerFile property).
+# See https://aka.ms/vscode-remote/containers/non-root-user for details.
+ARG USERNAME=vscode
+ARG USER_UID=1234
+ARG USER_GID=$USER_UID
+
+# Configure apt
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update 
+RUN apt-get upgrade -y 
+RUN apt-get -y install --no-install-recommends apt-utils dialog 2>&1 
+
+# Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user.
+RUN groupadd --gid $USER_GID $USERNAME 
+RUN useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME 
+# [Optional] Add sudo support for the non-root user
+RUN apt-get install -y sudo 
+RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME
+RUN chmod 0440 /etc/sudoers.d/$USERNAME 
+
+# Verify git, needed tools installed
+RUN apt-get -y install git openssh-client less iproute2 procps curl lsb-release zip unzip sed kafkacat telnet
+
+#-------------------Install SDKMan----------------------------------
+RUN curl -s https://get.sdkman.io | bash
+RUN chmod a+x "$HOME/.sdkman/bin/sdkman-init.sh"
+#-------------------------------------------------------------------------------------------------------------
+
+#-------------------Install Maven CLI Tools----------------------------------
+ARG MAVEN_VERSION=3.6.3
+RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && sdk install maven ${MAVEN_VERSION}"
+#-------------------------------------------------------------------------------------------------------------
+
+# Install snmp (rsu-data-controller dependency)
+RUN apt-get -y install snmp
+
+#-------------------Install Kafka----------------------------------
+RUN mkdir ~/Downloads
+RUN curl "https://archive.apache.org/dist/kafka/3.6.1/kafka_2.12-3.6.1.tgz" -o ~/Downloads/kafka.tgz
+RUN mkdir ~/kafka \
+    && cd ~/kafka \
+    && tar -xvzf ~/Downloads/kafka.tgz --strip 1
+RUN echo "\nadvertised.listeners=PLAINTEXT://localhost:9092" >> ~/kafka/config/server.properties    
+#-------------------------------------------------------------------------------------------------------------
+
+# Clean up
+RUN apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+ENV DEBIAN_FRONTEND=dialog
+
+# Allow for a consistant java home location for settings - image is changing over time
+RUN if [ ! -d "/docker-java-home" ]; then ln -s "${JAVA_HOME}" /docker-java-home; fi
+
+COPY kafka /etc/init.d/kafka
+RUN chmod 775 /etc/init.d/kafka && update-rc.d kafka defaults
+
+COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
+RUN chmod +x ./docker-entrypoint.sh
+ENTRYPOINT ["./docker-entrypoint.sh"]

.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.154.0/containers/java-8
+{
+	"name": "JPO Deduplicator Java 21",
+	"dockerFile": "Dockerfile",
+	"overrideCommand": false,
+	"shutdownAction": "stopContainer",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash"
+			},
+			"extensions": [
+				"vscjava.vscode-java-pack",
+				"vscjava.vscode-java-debug",
+				"vscjava.vscode-maven",
+				"vscjava.vscode-java-dependency",
+				"vscjava.vscode-java-test",
+				"hbenl.vscode-test-explorer",
+				"ms-vscode.test-adapter-converter",
+				"esbenp.prettier-vscode",
+				"mhutchie.git-graph",
+				"tabnine.tabnine-vscode",
+				"redhat.java",
+				"redhat.vscode-commons"
+			]
+		}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [8080, 9090, 46753, 46800, 5555, 6666, 8090, 2181, 9092],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "bash .devcontainer/post-create.sh",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode"
+	"runArgs": [
+		"--network=host"
+	]
+}

.devcontainer/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+trap stop SIGTERM SIGINT SIGQUIT SIGHUP ERR
+
+start() {
+    echo "Container started, starting kafka..." >>/root/startup.txt
+    sudo service kafka start
+}
+
+stop() {
+    echo "Container stopped, performing cleanup..." >>/root/shutdown.txt
+    sudo service kafka stop
+    sleep 2
+    exit 0
+}
+
+start
+
+while
+    sleep 1 &
+    wait $!
+do :; done

.devcontainer/jpo-deduplicator.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {
+		"java.configuration.updateBuildConfiguration": "automatic",
+		"java.server.launchMode": "Standard"
+	}
+}

.devcontainer/kafka

@@ -0,0 +1,36 @@
+DAEMON_PATH=/root/kafka
+PATH=$PATH:$DAEMON_PATH/bin
+PATH=$PATH:/root/.sdkman/candidates/java/current/bin
+
+export LOG_DIR=/var/log/kafka
+
+# See how we were called.
+case "$1" in
+  start)
+        # Start daemon.
+        echo "Starting Kafka";
+        $DAEMON_PATH/bin/kafka-server-start.sh -daemon $DAEMON_PATH/config/server.properties
+        ;;
+  stop)
+        # Stop daemons.
+        echo "Shutting down Kafka";
+        $DAEMON_PATH/bin/kafka-server-stop.sh
+        ;;
+  restart)
+        $0 stop
+        sleep 2
+        $0 start
+        ;;
+  status)
+        pid=`ps ax | grep -i 'kafka.Kafka' | grep -v grep | awk '{print $1}'`
+        if [ -n "$pid" ]
+          then
+          echo "Kafka is Running as PID: $pid"
+        else
+          echo "Kafka is not Running"
+        fi
+        ;;
+  *)
+        echo "Usage: $0 {start|stop|restart|status}"
+        exit 1
+esac

.devcontainer/post-create.sh

@@ -0,0 +1,30 @@
+cd ~/kafka/
+KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)"
+
+bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c config/kraft/server.properties
+
+# start kafka
+bin/kafka-server-start.sh -daemon config/kraft/server.properties
+# wait 2 seconds for the server to start and be able to add partitions
+sleep 2s
+# add topics
+
+# MAP
+bin/kafka-topics.sh --create --topic "topic.OdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.ProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# SPaT
+bin/kafka-topics.sh --create --topic "topic.OdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.ProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# BSM
+bin/kafka-topics.sh --create --topic "topic.OdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# TIM
+bin/kafka-topics.sh --create --topic "topic.OdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "jpo-utils"]
 	path = jpo-utils
-	url = git@github.com:usdot-jpo-ode/jpo-utils.git
+	url = https://github.com/usdot-jpo-ode/jpo-utils.git

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+  "recommendations": [
+    "redhat.vscode-xml",
+    "vscjava.vscode-java-pack",
+    "redhat.java",
+    "vmware.vscode-boot-dev-pack"
+  ]
+}

Makefile

@@ -0,0 +1,48 @@
+default:
+	$(info Make target options:)
+	$(info `make start` to run the deduplicator)
+	$(info `make build` to build the deduplicator)
+	$(info `make stop` to stop the deduplicator)
+	$(info `make delete` to stop the deduplicator and remove the volumes)
+	$(info `make restart` to stop and then start the deduplicator)
+	$(info `make rebuild` to stop, delete, and then rebuild the containers)
+	$(info `make clean-build` to rebuild the containers without using the cache)
+
+.PHONY: start
+start:
+ifeq ("$(wildcard .env)", "")
+	$(error "ERROR: deduplicator Environment file `.env` not found in ${PWD}")
+endif
+ifeq ("$(wildcard ./jpo-utils/.env)", "")
+	$(error "ERROR: jpo-utils Environment file `.env` not found in ${PWD}")
+endif
+	docker compose up -d
+
+build:
+ifeq ("$(wildcard .env)", "")
+	$(error "ERROR: jpo-ode Environment file `.env` not found in ${PWD}")
+endif
+ifeq ("$(wildcard ./jpo-utils/.env)", "")
+	$(error "ERROR: jpo-utils Environment file `.env` not found in ${PWD}")
+endif
+	docker compose build
+
+.PHONY: stop
+stop:
+	docker compose down
+
+.PHONY: delete
+delete:
+	docker compose down -v
+
+.PHONY: restart
+restart:
+	$(MAKE) stop start
+
+.PHONY: rebuild
+rebuild:
+	$(MAKE) stop delete build start
+
+.PHONY: clean-build
+clean-build:
+	docker compose build --no-cache