docker-compose.yml

services:
  zoo1:
    image: confluentinc/cp-zookeeper:7.3.2
    hostname: zoo1
    container_name: zoo1
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888
    networks:
      - kafka-net

  kafka1:
    image: confluentinc/cp-kafka:7.3.2
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9999:9999"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT                                       
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9999
    depends_on:
      - zoo1
    networks:
      - kafka-net
    healthcheck:
      test: [ "CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka1:19092" ]
      interval: 30s
      timeout: 30s
      retries: 3

  kafkaconnect:
    image: confluentinc/cp-kafka-connect:7.3.2
    hostname: kafkaConnect
    container_name: kafkaConnect
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafkaConnect"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "connect-cluster"
      CONNECT_CONFIG_STORAGE_TOPIC: "connect-configs"
      CONNECT_OFFSET_STORAGE_TOPIC: "connect-offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "connect-status"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars,/usr/share/confluent-hub-components"
      # edit replication factor to 1
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
    depends_on:
      - kafka1
      - elasticsearch
    networks:
      - kafka-net
    volumes:
      - ./kafka/sink/elasticsearch-sink.json:/temp/elasticsearch-sink.json
    entrypoint:
      - bash
      - -c
      - |
        confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:14.1.2
        /etc/confluent/docker/run
  schema-registry:
    image: confluentinc/cp-schema-registry:7.3.2
    hostname: schemaRegistry
    container_name: schemaRegistry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      SCHEMA_REGISTRY_HOST_NAME: "schemaRegistry"
      SCHEMA_REGISTRY_LISTENERS: "http://schemaRegistry:8081"
    depends_on:
      - kafka1
      - kafkaconnect
      - zoo1
    networks:
      - kafka-net

  kafkaUI:
    image: provectuslabs/kafka-ui:latest
    container_name: kafkaUI
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: "local"
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka1:19092"
      KAFKA_CLUSTERS_0_ZOOKEEPER: "zoo1:2181"
      KAFKA_CLUSTERS_0_JMXPORT: "9999"
      KAFKA_CLUSTERS_0_JMXHOST: "kafka1"
      KAFKA_CLUSTERS_0_SECURITYPROTOCOL: "PLAINTEXT"
      KAFKA_CLUSTERS_0_SASLMETADATA: "false"
      KAFKA_CLUSTERS_0_SASLMECHANISM: "PLAIN"
      KAFKA_CLUSTERS_0_SASLUSERNAME: "admin"
      KAFKA_CLUSTERS_0_SASLPASSWORD: "admin"
    depends_on:
      - kafka1
    networks:
      - kafka-net 

  redis:
    image: redis
    container_name: redis
    ports:
      - "6379:6379"
    command: ["redis-server", "--appendonly", "yes"]

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.5.0
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
    ports:
      - "9200:9200"
    networks:
      - kafka-net

  kibana:
    image: docker.elastic.co/kibana/kibana:8.5.0
    container_name: kibana
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    ports:
      - "5601:5601"
    networks:
      - kafka-net

networks:
  kafka-net:
    name: kafka-net