docker-compose.yml

version: "3.7"
services:
  pinot-zookeeper:
    image: zookeeper:latest
    container_name: pinot-zookeeper
    networks:
      - pinot
    ports:
      - "2181:2181"
    restart: always
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
  kafka:
    # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
    # An important note about accessing Kafka from clients on other machines:
    # -----------------------------------------------------------------------
    #
    # The config used here exposes port 29092 for _external_ connections to the broker
    # i.e. those from _outside_ the docker network. This could be from the host machine
    # running docker, or maybe further afield if you've got a more complicated setup.
    # If the latter is true, you will need to change the value 'localhost' in
    # KAFKA_ADVERTISED_LISTENERS to one that is resolvable to the docker host from those
    # remote clients
    #
    # For connections _internal_ to the docker network, such as from other services
    # and components, use kafka:9092.
    #
    # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
    # "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
    #
    image: confluentinc/cp-kafka:latest
    hostname: kafka
    container_name: kafka
    restart: always
    networks:
      - pinot
    ports:
      - "29092:29092"
    environment:
      KAFKA_ZOOKEEPER_CONNECT: "pinot-zookeeper:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_HOST_NAME: "kafka"
      ALLOW_PLAINTEXT_LISTENER: yes
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      # potential fix for ksql
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
    depends_on:
      - pinot-zookeeper
  pinot-controller:
    image: apachepinot/pinot:latest
    command: "StartController -zkAddress pinot-zookeeper:2181"
    container_name: pinot-controller
    restart: always
    networks:
      - pinot
    ports:
      - "9000:9000"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
    healthcheck:
      test: curl --fail http://pinot-controller:9000/health || exit 1
      interval: 20s
      retries: 5
      start_period: 10s
      timeout: 10s
    depends_on:
      - pinot-zookeeper
  pinot-broker:
    image: apachepinot/pinot:latest
    command: "StartBroker -zkAddress pinot-zookeeper:2181"
    restart: always
    container_name: "pinot-broker"
    networks:
      - pinot
    ports:
      - "8099:8099"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
    depends_on:
      - pinot-controller
  pinot-server:
    image: apachepinot/pinot:latest
    command: "StartServer -zkAddress pinot-zookeeper:2181"
    restart: always
    container_name: "pinot-server"
    networks:
      - pinot
    ports:
      - "8098:8098"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
    depends_on:
      - pinot-broker
  datagen:
    container_name: datagen
    build: ./scripts/datagen
    volumes:
      - .:/app
    restart: always
    networks:
      - pinot
    depends_on:
      pinot-controller:
        condition: service_healthy
  # ksqldb-server:
  #   image: confluentinc/ksqldb-server:0.29.0
  #   hostname: ksqldb-server
  #   container_name: ksqldb-server
  #   networks:
  #     - pinot
  #   restart: always
  #   ports:
  #     - "8088:8088"
  #   environment:
  #     KSQL_LISTENERS: http://0.0.0.0:8088
  #     KSQL_BOOTSTRAP_SERVERS: kafka:9092
  #     KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
  #     KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
  # ksqldb-cli:
  #   image: confluentinc/ksqldb-cli:0.29.0
  #   container_name: ksqldb-cli
  #   networks:
  #     - pinot
  #   depends_on:
  #     - ksqldb-server
  #   entrypoint: /bin/sh
  #   tty: true
  streamlit:
    container_name: streamlit
    build: ./streamlit
    restart: always
    volumes:
      - .:/app
    networks:
      - pinot
    depends_on:
      - datagen
    ports:
      - "8501:8501"
networks:
  pinot:
    name: pinot
    driver: bridge