juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -Dcom.sun.management.jmxremote=true
  28         -Dcom.sun.management.jmxremote.port=9101
  29         -Dcom.sun.management.jmxremote.authenticate=false
  30     volumes:
  31       - kafka-1-data:/var/lib/kafka/data
  32     ports:
  33       - 9081:9081
  34       - 9101:9101
  35     stop_grace_period: 120s
  36     depends_on:
  37       - zookeeper
  38
  39   kafka-2:
  40     image: confluentinc/cp-kafka:7.5.1
  41     environment:
  42       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  43       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  44       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  45       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  46       KAFKA_BROKER_ID: 2
  47       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  48       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  49       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  50       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  51       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  52       KAFKA_OPTS:
  53         -Dcom.sun.management.jmxremote=true
  54         -Dcom.sun.management.jmxremote.port=9102
  55         -Dcom.sun.management.jmxremote.authenticate=false
  56     volumes:
  57       - kafka-2-data:/var/lib/kafka/data
  58     ports:
  59       - 9092:9082
  60       - 9082:9082
  61       - 9102:9102
  62     networks:
  63       default:
  64         aliases:
  65           - kafka
  66     stop_grace_period: 120s
  67     depends_on:
  68       - zookeeper
  69
  70   kafka-3:
  71     image: confluentinc/cp-kafka:7.5.1
  72     environment:
  73       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  74       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  75       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  76       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  77       KAFKA_BROKER_ID: 3
  78       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  79       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  80       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  81       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  82       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  83       KAFKA_OPTS:
  84         -Dcom.sun.management.jmxremote=true
  85         -Dcom.sun.management.jmxremote.port=9103
  86         -Dcom.sun.management.jmxremote.authenticate=false
  87     volumes:
  88       - kafka-3-data:/var/lib/kafka/data
  89     ports:
  90       - 9083:9083
  91       - 9103:9103
  92     stop_grace_period: 120s
  93     depends_on:
  94       - zookeeper
  95
  96   schema-registry:
  97     image: confluentinc/cp-schema-registry:7.5.1
  98     environment:
  99       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 100       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 101       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 102       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 103     ports:
 104       - 8085:8085
 105     depends_on:
 106       - kafka-1
 107       - kafka-2
 108       - kafka-3
 109
 110   connect:
 111     image: confluentinc/cp-kafka-connect:7.5.1
 112     environment:
 113       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 114       CONNECT_REST_PORT: 8083
 115       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 116       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 117       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 118       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 119       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 120       CONNECT_GROUP_ID: kafka-connect
 121       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 122       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 123       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 124       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 125       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 126       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 127       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 128       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 129       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 130       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 131       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 132       CONNECT_PLUGIN_PATH: /usr/share/java/
 133     ports:
 134       - 8083:8083
 135     depends_on:
 136       - schema-registry
 137
 138   cli:
 139     image: juplo/toolbox
 140     command: sleep infinity
 141     stop_grace_period: 0s
 142     depends_on:
 143       - kafka-1
 144       - kafka-2
 145       - kafka-3
 146
 147   setup:
 148     image: juplo/toolbox
 149     command:
 150       - bash
 151       - -c
 152       - |
 153         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 154         if [ -e INITIALIZED ]
 155         then
 156           echo -n Bereits konfiguriert:
 157           cat INITIALIZED
 158           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 159         else
 160           kafka-topics --bootstrap-server kafka:9092 \
 161                        --delete \
 162                        --if-exists \
 163                        --topic test
 164           kafka-topics --bootstrap-server kafka:9092 \
 165                        --create \
 166                        --topic test \
 167                        --partitions 2 \
 168                        --replication-factor 3 \
 169                        --config min.insync.replicas=2 \
 170           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 171           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
 172           && date > INITIALIZED
 173         fi
 174     stop_grace_period: 0s
 175     depends_on:
 176       - cli
 177
 178   zoonavigator:
 179     image: elkozmon/zoonavigator:1.1.2
 180     ports:
 181       - "8000:80"
 182     environment:
 183       HTTP_PORT: 80
 184       CONNECTION_JUPLO_NAME: juplo
 185       CONNECTION_JUPLO_CONN: zookeeper:2181
 186       AUTO_CONNECT_CONNECTION_ID: JUPLO
 187     depends_on:
 188       - zookeeper
 189
 190   akhq:
 191     image: tchiotludo/akhq:0.23.0
 192     ports:
 193       - 8888:8080
 194     environment:
 195       AKHQ_CONFIGURATION: |
 196         akhq:
 197           connections:
 198             docker-kafka-server:
 199               properties:
 200                 bootstrap.servers: "kafka:9092"
 201               schema-registry:
 202                 url: "http://schema-registry:8085"
 203               connect:
 204                 - name: "connect"
 205                   url: "http://connect:8083"
 206     depends_on:
 207       - kafka-1
 208       - kafka-2
 209       - kafka-3
 210
 211   producer:
 212     image: juplo/simple-producer:1.0-SNAPSHOT
 213     command: kafka:9092 test producer
 214
 215   consumer:
 216     image: juplo/simple-consumer:1.0-SNAPSHOT
 217     command: kafka:9092 test my-group consumer
 218
 219 volumes:
 220   zookeeper-data:
 221   zookeeper-log:
 222   kafka-1-data:
 223   kafka-2-data:
 224   kafka-3-data: