juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_OPTS:
  25         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  26         -Dcom.sun.management.jmxremote=true
  27         -Dcom.sun.management.jmxremote.port=9101
  28         -Dcom.sun.management.jmxremote.authenticate=false
  29     volumes:
  30       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  31       - kafka-1-data:/var/lib/kafka/data
  32     ports:
  33       - 9081:9081
  34     stop_grace_period: 120s
  35     depends_on:
  36       - zookeeper
  37
  38   kafka-2:
  39     image: confluentinc/cp-kafka:7.5.1
  40     environment:
  41       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  42       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  43       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  44       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  45       KAFKA_BROKER_ID: 2
  46       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  47       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  48       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  49       KAFKA_OPTS:
  50         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  51         -Dcom.sun.management.jmxremote=true
  52         -Dcom.sun.management.jmxremote.port=9101
  53         -Dcom.sun.management.jmxremote.authenticate=false
  54     volumes:
  55       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  56       - kafka-2-data:/var/lib/kafka/data
  57     ports:
  58       - 9092:9082
  59       - 9082:9082
  60     networks:
  61       default:
  62         aliases:
  63           - kafka
  64     stop_grace_period: 120s
  65     depends_on:
  66       - zookeeper
  67
  68   kafka-3:
  69     image: confluentinc/cp-kafka:7.5.1
  70     environment:
  71       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  72       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  73       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  74       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  75       KAFKA_BROKER_ID: 3
  76       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  77       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  78       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  79       KAFKA_OPTS:
  80         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  81         -Dcom.sun.management.jmxremote=true
  82         -Dcom.sun.management.jmxremote.port=9101
  83         -Dcom.sun.management.jmxremote.authenticate=false
  84     volumes:
  85       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  86       - kafka-3-data:/var/lib/kafka/data
  87     ports:
  88       - 9083:9083
  89     stop_grace_period: 120s
  90     depends_on:
  91       - zookeeper
  92
  93   schema-registry:
  94     image: confluentinc/cp-schema-registry:7.5.1
  95     environment:
  96       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
  97       SCHEMA_REGISTRY_HOST_NAME: schema-registry
  98       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
  99       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 100     ports:
 101       - 8085:8085
 102     depends_on:
 103       - kafka-1
 104       - kafka-2
 105       - kafka-3
 106
 107   connect:
 108     image: confluentinc/cp-kafka-connect:7.5.1
 109     environment:
 110       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 111       CONNECT_REST_PORT: 8083
 112       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 113       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 114       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 115       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 116       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 117       CONNECT_GROUP_ID: kafka-connect
 118       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 119       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 120       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 121       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 122       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 123       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 124       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 125       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 126       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 127       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 128       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 129       CONNECT_PLUGIN_PATH: /usr/share/java/
 130     ports:
 131       - 8083:8083
 132     depends_on:
 133       - schema-registry
 134
 135   cli:
 136     image: juplo/toolbox
 137     command: sleep infinity
 138     stop_grace_period: 0s
 139     depends_on:
 140       - kafka-1
 141       - kafka-2
 142       - kafka-3
 143
 144   setup:
 145     image: juplo/toolbox
 146     command:
 147       - bash
 148       - -c
 149       - |
 150         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 151         if [ -e INITIALIZED ]
 152         then
 153           echo -n Bereits konfiguriert:
 154           cat INITIALIZED
 155           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 156         else
 157           kafka-topics --bootstrap-server kafka:9092 \
 158                        --delete \
 159                        --if-exists \
 160                        --topic test
 161           kafka-topics --bootstrap-server kafka:9092 \
 162                        --create \
 163                        --topic test \
 164                        --partitions 2 \
 165                        --replication-factor 3 \
 166                        --config min.insync.replicas=2 \
 167           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 168           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
 169           && date > INITIALIZED
 170         fi
 171     stop_grace_period: 0s
 172     depends_on:
 173       - cli
 174
 175   akhq:
 176     image: tchiotludo/akhq:0.23.0
 177     ports:
 178       - 8888:8080
 179     environment:
 180       AKHQ_CONFIGURATION: |
 181         akhq:
 182           connections:
 183             docker-kafka-server:
 184               properties:
 185                 bootstrap.servers: "kafka:9092"
 186               schema-registry:
 187                 url: "http://schema-registry:8085"
 188               connect:
 189                 - name: "connect"
 190                   url: "http://connect:8083"
 191     depends_on:
 192       - kafka-1
 193       - kafka-2
 194       - kafka-3
 195
 196 volumes:
 197   zookeeper-data:
 198   zookeeper-log:
 199   kafka-1-data:
 200   kafka-2-data:
 201   kafka-3-data: