docker-compose-cdc.yml

x-kafka-common:
  &kafka-common
  # profiles: [ "kafka" ]
  image: confluentinc/cp-kafka:7.4.1
  environment:
    &kafka-common-env
    KAFKA_ZOOKEEPER_CONNECT: "zk:2181"
    KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
    KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
    KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
    KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
    KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT

  depends_on:
    &kafka-common-depends
    zk:
      condition: service_healthy

x-connect-common:
  &connect-common
  # profiles: [ "kafka" ]
  build:
    dockerfile: ./docker/debezium/Dockerfile
    args:
      DEBEZIUM_VERSION: 2.3.3.Final
      CONFLUENT_VERSION: 7.4.1
  image: 1ambda/lakehouse:debezium-connect-2.3.3.Final
  environment:
    &connect-common-env
    # https://hub.docker.com/r/debezium/connect-base
    BOOTSTRAP_SERVERS: kafka1:29092
    GROUP_ID: cdc.inventory
    CONFIG_STORAGE_TOPIC: connect-cluster.inventory.config
    OFFSET_STORAGE_TOPIC: connect-cluster.inventory.offset
    STATUS_STORAGE_TOPIC: connect-cluster.inventory.status
    CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
    CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
    CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
    CONNECT_STATUS_STORAGE_PARTITIONS: 1
    CONNECT_CONFIG_STORAGE_PARTITIONS: 1
    CONNECT_OFFSET_STORAGE_PARTITIONS: 1
    OFFSET_FLUSH_INTERVAL_MS: 10000
    OFFSET_FLUSH_TIMEOUT_MS: 5000
    SHUTDOWN_TIMEOUT: 30
    LOG_LEVEL: INFO
    CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: All
    CONNECT_ACCESS_CONTROL_ALLOW_ORIGIN: "*"
    CONNECT_ACCESS_CONTROL_ALLOW_METHODS: GET,POST,PUT,DELETE,OPTIONS
    CONNECT_ACCESS_CONTROL_ALLOW_HEADERS: origin,content-type,accept,authorization
    CONNECT_LISTENERS: http://0.0.0.0:8083
    CONNECT_REST_PORT: 8083
    CONNECT_REST_ADVERTISED_PORT: 8083
    # - HEAP_OPTS=-Xms2G -Xmx2G
    # - CONNECT_PRODUCER_BUFFER_MEMORY=45554432
    # - CONNECT_MAX_REQUEST_SIZE=10485760
  depends_on:
    &connect-common-depends
    mysql:
      condition: service_healthy
    kafka1:
      condition: service_healthy

x-zk-common:
  &zk-common
  image: confluentinc/cp-zookeeper:7.4.1
  # profiles: [ "kafka" ]
  environment:
    &zk-common-env
    ZOOKEEPER_SERVERS: zk:2888:3888

services:
  ####################################################################################################
  # Kafka
  ####################################################################################################
  zk:
    <<: *zk-common
    hostname: zk
    container_name: zk
    ports:
      - "2181:2181"
    environment:
      <<: *zk-common-env
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
    healthcheck:
      test: nc -z localhost 2181 || exit -1
      start_period: 15s
      interval: 10s
      timeout: 10s
      retries: 3
    restart: always
    # volumes:
    #   - ./docker/volume/zookeeper/data:/var/lib/zookeeper/data
    #   - ./docker/volume/zookeeper/log:/var/lib/zookeeper/log

  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:7.4.1
    hostname: kafka-schema-registry
    container_name: kafka-schema-registry
    ports:
      - 8085:8085
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:29092
      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      # SCHEMA_REGISTRY_LISTENERS: http://kafka-schema-registry:8085
      SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8085"
      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http"
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
    depends_on:
      kafka1:
        condition: service_healthy

  kafka1:
    <<: *kafka-common
    hostname: kafka1
    container_name: kafka1
    ports:
      - "9092:9092"
      - "29092:29092"
      - "9997:9997"
    environment:
      <<: *kafka-common-env
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
      KAFKA_JMX_PORT: 9997
      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997
    healthcheck:
      test: nc -z localhost 9092 || exit -1
      start_period: 15s
      interval: 10s
      timeout: 10s
      retries: 3
    restart: always
    depends_on:
      *kafka-common-depends
    # volumes:
    #   - ./docker/volume/kafka/kafka1-data:/var/lib/kafka/data

  kafka-ui:
    # profiles: [ "kafka" ]
    image: provectuslabs/kafka-ui
    hostname: kafka-ui
    container_name: kafka-ui
    ports:
      - "8088:8080"
    # 환경 변수 설정
    environment:
      - DYNAMIC_CONFIG_ENABLED=true
      - KAFKA_CLUSTERS_0_NAME=local
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka1:29092 #,kafka2:29093
      - KAFKA_CLUSTERS_0_ZOOKEEPER=zk:2181
      - KAFKA_CLUSTERS_0_METRICS_PORT=9997
      - KAFKA_CLUSTERS_0_SCHEMAREGISTRY=http://kafka-schema-registry:8085
      - KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME=connect-cdc
      - KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS=http://kafka-connect-1:8083 #,http://kafka-connect-2:8083
    restart: always
    depends_on:
      kafka1:
        condition: service_healthy


  ####################################################################################################
  # MySQL
  ####################################################################################################
  mysql:
    # profiles: [ "kafka" ]
    image: mysql:5.7
    platform: linux/amd64
    container_name: mysql
    hostname: mysql
    ports:
      - "3306:3306"
    environment:
      TZ: UTC
      MYSQL_ROOT_USER: "root"
      MYSQL_ROOT_PASSWORD: "admin"
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_USER: "mysqluser"
      MYSQL_PASSWORD: "mysqlpw"
    healthcheck:
      test: [ "CMD-SHELL", "mysqladmin -u $$MYSQL_USER -p$$MYSQL_PASSWORD ping -h localhost || exit 1" ]
      start_period: 30s
      interval: 15s
      timeout: 15s
      retries: 3
    restart: always
    volumes:
      - ./docker/mysql/my.cnf:/etc/mysql/conf.d/my.cnf
      - ./docker/mysql/entrypoint.sql:/docker-entrypoint-initdb.d/entrypoint.sql
      # - ./docker/volume/mysql/data:/var/lib/mysql
      # - ./docker/volume/mysql/logs/:/var/log/mysql


  ####################################################################################################
  # Kafka Producer: Debezium MySQL Connector (Connect Distributed Mode)
  ####################################################################################################

  kafka-connect-1:
    <<: *connect-common
    container_name: kafka-connect-1
    hostname: kafka-connect-1
    ports:
      - "8083:8083"
    environment:
      <<: *connect-common-env
      CONNECT_REST_HOST_NAME: kafka-connect-1
      CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect-1
    restart: always
    depends_on:
      *connect-common-depends

# Configure Network
networks:
  default:
    name: lakehouse