Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Kafka version update #59

Merged
merged 7 commits into from
Feb 22, 2024
+90 −131
Merged

Kafka version update #59

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
431e3b9
docker-compose update
mwodahl Feb 15, 2024
ba153ea
Merge remote-tracking branch 'origin/dev' into Update/kafka-version
mwodahl Feb 15, 2024
e9fa119
Remove unnecessary ports
mwodahl Feb 21, 2024
094a8c0
Update quickstart-compose.yml
mwodahl Feb 21, 2024
c85ff64
.devcontainer update
mwodahl Feb 21, 2024
3c81180
docs update
mwodahl Feb 21, 2024
12e9004
add /kafka/ directory to .gitignore
mwodahl Feb 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 23 additions & 25 deletions docker-compose-ppm-nsv.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,34 @@

version: '3'
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"

kafka:
image: wurstmeister/kafka
image: bitnami/kafka:latest
hostname: kafka
ports:
- "9092:9092"
- "9093:9093"
- "9094:9094"
volumes:
- "${DOCKER_SHARED_VOLUME}:/bitnami"
environment:
DOCKER_HOST_IP: ${DOCKER_HOST_IP}
ZK: ${DOCKER_HOST_IP}:2181
KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ENABLE_KRAFT: "yes"
KAFKA_CFG_PROCESS_ROLES: "broker,controller"
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
KAFKA_CFG_LISTENERS: "PLAINTEXT://:9094,CONTROLLER://:9093,EXTERNAL://:9092"
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9094,EXTERNAL://${DOCKER_HOST_IP}:9092"
KAFKA_BROKER_ID: "1"
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
ALLOW_PLAINTEXT_LISTENER: "yes"
KAFKA_CFG_NODE_ID: "1"
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_CREATE_TOPICS: "topic.OdeBsmPojo:1:1,topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdeTimBroadcastJson:1:1,topic.J2735TimBroadcastJson:1:1,topic.OdeDriverAlertJson:1:1,topic.Asn1DecoderInput:1:1,topic.Asn1DecoderOutput:1:1,topic.Asn1EncoderInput:1:1,topic.Asn1EncoderOutput:1:1,topic.SDWDepositorInput:1:1"
KAFKA_DELETE_TOPIC_ENABLED: "true"
KAFKA_CLEANUP_POLICY: "delete" # delete old logs
KAFKA_LOG_RETENTION_HOURS: 2
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000
KAFKA_RETENTION_MS: 7200000 # delete old logs after 2 hours
KAFKA_SEGMENT_MS: 7200000 # roll segment logs every 2 hours.
# This configuration controls the period of time after
# which Kafka will force the log to roll even if the segment
# file isn't full to ensure that retention can delete or compact old data.
depends_on:
- zookeeper
volumes:
- ${DOCKER_SHARED_VOLUME}/var/run/docker.sock:/var/run/docker.sock
KAFKA_CREATE_TOPICS: "topic.OdeBsmPojo:1:1,topic.OdeSpatTxPojo:1:1,topic.OdeSpatPojo:1:1,topic.OdeSpatJson:1:1,topic.FilteredOdeSpatJson:1:1,topic.OdeSpatRxJson:1:1,topic.OdeSpatRxPojo:1:1,topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdeTimBroadcastJson:1:1,topic.J2735TimBroadcastJson:1:1,topic.OdeDriverAlertJson:1:1,topic.Asn1DecoderInput:1:1,topic.Asn1DecoderOutput:1:1,topic.Asn1EncoderInput:1:1,topic.Asn1EncoderOutput:1:1,topic.SDWDepositorInput:1:1,topic.OdeTIMCertExpirationTimeJson:1:1,topic.OdeRawEncodedBSMJson:1:1,topic.OdeRawEncodedSPATJson:1:1,topic.OdeRawEncodedTIMJson:1:1,topic.OdeRawEncodedMAPJson:1:1,topic.OdeMapTxPojo:1:1,topic.OdeMapJson:1:1,topic.OdeRawEncodedSSMJson:1:1,topic.OdeSsmPojo:1:1,topic.OdeSsmJson:1:1,topic.OdeRawEncodedSRMJson:1:1,topic.OdeSrmTxPojo:1:1,topic.OdeSrmJson:1:1,topic.OdeRawEncodedPSMJson:1:1,topic.OdePsmTxPojo:1:1,topic.OdePsmJson:1:1"
KAFKA_CFG_DELETE_TOPIC_ENABLE: "true"
KAFKA_CFG_LOG_RETENTION_HOURS: 2
logging:
options:
max-size: "10m"
max-file: "5"

ode:
build: .
Expand Down Expand Up @@ -193,7 +192,6 @@ services:
SDW_PASSWORD: ${SDW_PASSWORD}
depends_on:
- kafka
- zookeeper
- ode

sec:
Expand Down
48 changes: 19 additions & 29 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,32 @@
version: '3'
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
logging:
options:
max-size: "10m"
max-file: "5"

kafka:
image: wurstmeister/kafka
image: bitnami/kafka:latest
hostname: kafka
ports:
- "9092:9092"
- "9093:9093"
- "9094:9094"
volumes:
- "${DOCKER_SHARED_VOLUME}:/bitnami"
environment:
DOCKER_HOST_IP: ${DOCKER_HOST_IP}
ZK: ${DOCKER_HOST_IP}:2181
KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ENABLE_KRAFT: "yes"
KAFKA_CFG_PROCESS_ROLES: "broker,controller"
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
KAFKA_CFG_LISTENERS: "PLAINTEXT://:9094,CONTROLLER://:9093,EXTERNAL://:9092"
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9094,EXTERNAL://${DOCKER_HOST_IP}:9092"
KAFKA_BROKER_ID: "1"
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
ALLOW_PLAINTEXT_LISTENER: "yes"
KAFKA_CFG_NODE_ID: "1"
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_CREATE_TOPICS: "topic.OdeBsmPojo:1:1,topic.OdeSpatTxPojo:1:1,topic.OdeSpatPojo:1:1,topic.OdeSpatJson:1:1,topic.FilteredOdeSpatJson:1:1,topic.OdeSpatRxJson:1:1,topic.OdeSpatRxPojo:1:1,topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdeTimBroadcastJson:1:1,topic.J2735TimBroadcastJson:1:1,topic.OdeDriverAlertJson:1:1,topic.Asn1DecoderInput:1:1,topic.Asn1DecoderOutput:1:1,topic.Asn1EncoderInput:1:1,topic.Asn1EncoderOutput:1:1,topic.SDWDepositorInput:1:1,topic.OdeTIMCertExpirationTimeJson:1:1,topic.OdeRawEncodedBSMJson:1:1,topic.OdeRawEncodedSPATJson:1:1,topic.OdeRawEncodedTIMJson:1:1,topic.OdeRawEncodedMAPJson:1:1,topic.OdeMapTxPojo:1:1,topic.OdeMapJson:1:1,topic.OdeRawEncodedSSMJson:1:1,topic.OdeSsmPojo:1:1,topic.OdeSsmJson:1:1,topic.OdeRawEncodedSRMJson:1:1,topic.OdeSrmTxPojo:1:1,topic.OdeSrmJson:1:1,topic.OdeRawEncodedPSMJson:1:1,topic.OdePsmTxPojo:1:1,topic.OdePsmJson:1:1"
KAFKA_DELETE_TOPIC_ENABLED: "true"
KAFKA_CLEANUP_POLICY: "delete" # delete old logs
KAFKA_LOG_RETENTION_HOURS: 2
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000
KAFKA_RETENTION_MS: 7200000 # delete old logs after 2 hours
KAFKA_SEGMENT_MS: 7200000 # roll segment logs every 2 hours.
# This configuration controls the period of time after
# which Kafka will force the log to roll even if the segment
# file isn't full to ensure that retention can delete or compact old data.
depends_on:
- zookeeper
volumes:
- ${DOCKER_SHARED_VOLUME}/var/run/docker.sock:/var/run/docker.sock
KAFKA_CFG_DELETE_TOPIC_ENABLE: "true"
KAFKA_CFG_LOG_RETENTION_HOURS: 2
logging:
options:
max-size: "10m"
max-size: "10m"
max-file: "5"

ode:
Expand Down Expand Up @@ -254,7 +245,6 @@ services:
SDW_API_KEY: ${SDW_API_KEY}
depends_on:
- kafka
- zookeeper
- ode
logging:
options:
Expand Down
Loading