Merge pull request #1 from usdot-jpo-ode/deduplicator-setup

Copying over deduplicator from jpo-utils

.devcontainer/Dockerfile

@@ -0,0 +1,69 @@
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+
+FROM eclipse-temurin:21-jdk
+
+# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser"
+# property in devcontainer.json to use it. On Linux, the container user's GID/UIDs
+# will be updated to match your local UID/GID (when using the dockerFile property).
+# See https://aka.ms/vscode-remote/containers/non-root-user for details.
+ARG USERNAME=vscode
+ARG USER_UID=1234
+ARG USER_GID=$USER_UID
+
+# Configure apt
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update 
+RUN apt-get upgrade -y 
+RUN apt-get -y install --no-install-recommends apt-utils dialog 2>&1 
+
+# Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user.
+RUN groupadd --gid $USER_GID $USERNAME 
+RUN useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME 
+# [Optional] Add sudo support for the non-root user
+RUN apt-get install -y sudo 
+RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME
+RUN chmod 0440 /etc/sudoers.d/$USERNAME 
+
+# Verify git, needed tools installed
+RUN apt-get -y install git openssh-client less iproute2 procps curl lsb-release zip unzip sed kafkacat telnet
+
+#-------------------Install SDKMan----------------------------------
+RUN curl -s https://get.sdkman.io | bash
+RUN chmod a+x "$HOME/.sdkman/bin/sdkman-init.sh"
+#-------------------------------------------------------------------------------------------------------------
+
+#-------------------Install Maven CLI Tools----------------------------------
+ARG MAVEN_VERSION=3.6.3
+RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && sdk install maven ${MAVEN_VERSION}"
+#-------------------------------------------------------------------------------------------------------------
+
+# Install snmp (rsu-data-controller dependency)
+RUN apt-get -y install snmp
+
+#-------------------Install Kafka----------------------------------
+RUN mkdir ~/Downloads
+RUN curl "https://archive.apache.org/dist/kafka/3.6.1/kafka_2.12-3.6.1.tgz" -o ~/Downloads/kafka.tgz
+RUN mkdir ~/kafka \
+    && cd ~/kafka \
+    && tar -xvzf ~/Downloads/kafka.tgz --strip 1
+RUN echo "\nadvertised.listeners=PLAINTEXT://localhost:9092" >> ~/kafka/config/server.properties    
+#-------------------------------------------------------------------------------------------------------------
+
+# Clean up
+RUN apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+ENV DEBIAN_FRONTEND=dialog
+
+# Allow for a consistant java home location for settings - image is changing over time
+RUN if [ ! -d "/docker-java-home" ]; then ln -s "${JAVA_HOME}" /docker-java-home; fi
+
+COPY kafka /etc/init.d/kafka
+RUN chmod 775 /etc/init.d/kafka && update-rc.d kafka defaults
+
+COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
+RUN chmod +x ./docker-entrypoint.sh
+ENTRYPOINT ["./docker-entrypoint.sh"]

.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.154.0/containers/java-8
+{
+	"name": "JPO Deduplicator Java 21",
+	"dockerFile": "Dockerfile",
+	"overrideCommand": false,
+	"shutdownAction": "stopContainer",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash"
+			},
+			"extensions": [
+				"vscjava.vscode-java-pack",
+				"vscjava.vscode-java-debug",
+				"vscjava.vscode-maven",
+				"vscjava.vscode-java-dependency",
+				"vscjava.vscode-java-test",
+				"hbenl.vscode-test-explorer",
+				"ms-vscode.test-adapter-converter",
+				"esbenp.prettier-vscode",
+				"mhutchie.git-graph",
+				"tabnine.tabnine-vscode",
+				"redhat.java",
+				"redhat.vscode-commons"
+			]
+		}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [8080, 9090, 46753, 46800, 5555, 6666, 8090, 2181, 9092],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "bash .devcontainer/post-create.sh",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode"
+	"runArgs": [
+		"--network=host"
+	]
+}

.devcontainer/docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+trap stop SIGTERM SIGINT SIGQUIT SIGHUP ERR
+
+start() {
+    echo "Container started, starting kafka..." >>/root/startup.txt
+    sudo service kafka start
+}
+
+stop() {
+    echo "Container stopped, performing cleanup..." >>/root/shutdown.txt
+    sudo service kafka stop
+    sleep 2
+    exit 0
+}
+
+start
+
+while
+    sleep 1 &
+    wait $!
+do :; done

.devcontainer/jpo-deduplicator.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {
+		"java.configuration.updateBuildConfiguration": "automatic",
+		"java.server.launchMode": "Standard"
+	}
+}

.devcontainer/kafka

@@ -0,0 +1,36 @@
+DAEMON_PATH=/root/kafka
+PATH=$PATH:$DAEMON_PATH/bin
+PATH=$PATH:/root/.sdkman/candidates/java/current/bin
+
+export LOG_DIR=/var/log/kafka
+
+# See how we were called.
+case "$1" in
+  start)
+        # Start daemon.
+        echo "Starting Kafka";
+        $DAEMON_PATH/bin/kafka-server-start.sh -daemon $DAEMON_PATH/config/server.properties
+        ;;
+  stop)
+        # Stop daemons.
+        echo "Shutting down Kafka";
+        $DAEMON_PATH/bin/kafka-server-stop.sh
+        ;;
+  restart)
+        $0 stop
+        sleep 2
+        $0 start
+        ;;
+  status)
+        pid=`ps ax | grep -i 'kafka.Kafka' | grep -v grep | awk '{print $1}'`
+        if [ -n "$pid" ]
+          then
+          echo "Kafka is Running as PID: $pid"
+        else
+          echo "Kafka is not Running"
+        fi
+        ;;
+  *)
+        echo "Usage: $0 {start|stop|restart|status}"
+        exit 1
+esac

.devcontainer/post-create.sh

@@ -0,0 +1,30 @@
+cd ~/kafka/
+KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)"
+
+bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c config/kraft/server.properties
+
+# start kafka
+bin/kafka-server-start.sh -daemon config/kraft/server.properties
+# wait 2 seconds for the server to start and be able to add partitions
+sleep 2s
+# add topics
+
+# MAP
+bin/kafka-topics.sh --create --topic "topic.OdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeMapJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.ProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedMap" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# SPaT
+bin/kafka-topics.sh --create --topic "topic.OdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeSpatJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.ProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedProcessedSpat" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# BSM
+bin/kafka-topics.sh --create --topic "topic.OdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeBsmJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+
+# TIM
+bin/kafka-topics.sh --create --topic "topic.OdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
+bin/kafka-topics.sh --create --topic "topic.DeduplicatedOdeTimJson" --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1

.github/workflows/artifact-publish.yml

@@ -0,0 +1,48 @@
+name: Publish Java Package
+
+on:
+  push:
+    branches:
+      - "develop"
+      - "master"
+      - "release/*"
+
+
+jobs:
+  deduplicator-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up JDK 21a
+      uses: actions/setup-java@v4
+      with:
+        java-version: '21'
+        distribution: 'temurin'
+
+    - name: Remove snapshot from version
+      run: |
+        cd jpo-deduplicator
+        mvn versions:set -DremoveSnapshot
+
+    - name: Build with Maven
+      env:
+        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        MAVEN_GITHUB_TOKEN_NAME: ${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+        MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+        MAVEN_GITHUB_ORG: ${{ github.repository_owner }}
+      run: |
+        cd jpo-deduplicator
+        mvn -s settings.xml -B package --file pom.xml -DskipTests -Ppackage-jar
+
+    - name: Publish to GitHub Packages
+      run: |
+        cd jpo-deduplicator
+        mvn -s settings.xml --batch-mode -Dgithub_organization=${{ github.repository_owner }} deploy -DskipTests -Ppackage-jar
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        MAVEN_GITHUB_TOKEN_NAME: ${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+        MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+        MAVEN_GITHUB_ORG: ${{ github.repository_owner }}

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  pull_request:
+    paths:
+      - 'jpo-deduplicator/**'
+
+
+
+jobs:  
+  deduplicator-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: "21"
+          distribution: "temurin"
+          cache: 'maven'
+
+      - name: Run tests
+        env:
+          MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          MAVEN_GITHUB_ORG: ${{ github.repository_owner }}
+        run: |
+          cd $GITHUB_WORKSPACE/jpo-deduplicator
+          mvn verify -s ./settings.xml

.github/workflows/docker.yml

@@ -0,0 +1,26 @@
+name: Docker build
+
+on:
+  pull_request:
+      types: [opened, synchronize, reopened]
+
+jobs:
+  jpo-deduplicator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          build-args: |
+            MAVEN_GITHUB_TOKEN_NAME=${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+            MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }}
+            MAVEN_GITHUB_ORG=${{ github.repository_owner }}
+          secrets: |
+            MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/dockerhub.yml

@@ -0,0 +1,41 @@
+name: "DockerHub Build and Push"
+
+on:
+  push:
+    branches:
+      - "develop"
+      - "master"
+      - "release/*"
+
+jobs:
+  dockerhub-jpo-deduplicator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Replcae Docker tag
+        id: set_tag
+        run: echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g')" >> $GITHUB_ENV
+
+      - name: Build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: usdotjpoode/jpo-deduplicator:${{ env.TAG }}
+          build-args: |
+            MAVEN_GITHUB_TOKEN_NAME=${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+            MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }}
+            MAVEN_GITHUB_ORG=${{ github.repository_owner }}
+          secrets: |
+            MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -0,0 +1,3 @@
+**/.env
+
+**/target

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "jpo-utils"]
+	path = jpo-utils
+	url = https://github.com/usdot-jpo-ode/jpo-utils.git

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+  "recommendations": [
+    "redhat.vscode-xml",
+    "vscjava.vscode-java-pack",
+    "redhat.java",
+    "vmware.vscode-boot-dev-pack"
+  ]
+}