juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31     volumes:
  32       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  33       - kafka-1-data:/var/lib/kafka/data
  34     ports:
  35       - 9081:9081
  36     stop_grace_period: 120s
  37     depends_on:
  38       - zookeeper
  39
  40   kafka-2:
  41     image: confluentinc/cp-kafka:7.5.1
  42     environment:
  43       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  44       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  45       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  46       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  47       KAFKA_BROKER_ID: 2
  48       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  49       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  50       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  51       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  52       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  53       KAFKA_OPTS:
  54         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  55         -Dcom.sun.management.jmxremote=true
  56         -Dcom.sun.management.jmxremote.port=9101
  57         -Dcom.sun.management.jmxremote.authenticate=false
  58     volumes:
  59       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  60       - kafka-2-data:/var/lib/kafka/data
  61     ports:
  62       - 9092:9082
  63       - 9082:9082
  64     networks:
  65       default:
  66         aliases:
  67           - kafka
  68     stop_grace_period: 120s
  69     depends_on:
  70       - zookeeper
  71
  72   kafka-3:
  73     image: confluentinc/cp-kafka:7.5.1
  74     environment:
  75       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  76       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  77       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  78       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  79       KAFKA_BROKER_ID: 3
  80       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  81       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  82       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  83       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  84       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  85       KAFKA_OPTS:
  86         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  87         -Dcom.sun.management.jmxremote=true
  88         -Dcom.sun.management.jmxremote.port=9101
  89         -Dcom.sun.management.jmxremote.authenticate=false
  90     volumes:
  91       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  92       - kafka-3-data:/var/lib/kafka/data
  93     ports:
  94       - 9083:9083
  95     stop_grace_period: 120s
  96     depends_on:
  97       - zookeeper
  98
  99   schema-registry:
 100     image: confluentinc/cp-schema-registry:7.5.1
 101     environment:
 102       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 103       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 104       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 105       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 106     ports:
 107       - 8085:8085
 108     depends_on:
 109       - kafka-1
 110       - kafka-2
 111       - kafka-3
 112
 113   connect:
 114     image: confluentinc/cp-kafka-connect:7.5.1
 115     environment:
 116       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 117       CONNECT_REST_PORT: 8083
 118       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 119       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 120       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 121       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 122       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 123       CONNECT_GROUP_ID: kafka-connect
 124       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 125       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 126       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 127       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 128       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 129       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 130       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 131       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 132       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 133       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 134       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 135       CONNECT_PLUGIN_PATH: /usr/share/java/
 136     ports:
 137       - 8083:8083
 138     depends_on:
 139       - schema-registry
 140
 141   cli:
 142     image: juplo/toolbox
 143     command: sleep infinity
 144     stop_grace_period: 0s
 145     depends_on:
 146       - kafka-1
 147       - kafka-2
 148       - kafka-3
 149
 150   setup:
 151     image: juplo/toolbox
 152     command:
 153       - bash
 154       - -c
 155       - |
 156         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 157         if [ -e INITIALIZED ]
 158         then
 159           echo -n Bereits konfiguriert:
 160           cat INITIALIZED
 161           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 162         else
 163           kafka-topics --bootstrap-server kafka:9092 \
 164                        --delete \
 165                        --if-exists \
 166                        --topic test
 167           kafka-topics --bootstrap-server kafka:9092 \
 168                        --create \
 169                        --topic test \
 170                        --partitions 2 \
 171                        --replication-factor 3 \
 172                        --config min.insync.replicas=2 \
 173           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 174           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
 175           && date > INITIALIZED
 176         fi
 177     stop_grace_period: 0s
 178     depends_on:
 179       - cli
 180
 181   zoonavigator:
 182     image: elkozmon/zoonavigator:1.1.2
 183     ports:
 184       - "8000:80"
 185     environment:
 186       HTTP_PORT: 80
 187       CONNECTION_JUPLO_NAME: juplo
 188       CONNECTION_JUPLO_CONN: zookeeper:2181
 189       AUTO_CONNECT_CONNECTION_ID: JUPLO
 190     depends_on:
 191       - zookeeper
 192
 193   akhq:
 194     image: tchiotludo/akhq:0.23.0
 195     ports:
 196       - 8888:8080
 197     environment:
 198       AKHQ_CONFIGURATION: |
 199         akhq:
 200           connections:
 201             docker-kafka-server:
 202               properties:
 203                 bootstrap.servers: "kafka:9092"
 204               schema-registry:
 205                 url: "http://schema-registry:8085"
 206               connect:
 207                 - name: "connect"
 208                   url: "http://connect:8083"
 209     depends_on:
 210       - kafka-1
 211       - kafka-2
 212       - kafka-3
 213
 214   producer:
 215     image: juplo/simple-producer:1.0-SNAPSHOT
 216     command: kafka:9092 test producer
 217
 218 volumes:
 219   zookeeper-data:
 220   zookeeper-log:
 221   kafka-1-data:
 222   kafka-2-data:
 223   kafka-3-data: