juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31     volumes:
  32       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  33       - kafka-1-data:/var/lib/kafka/data
  34     ports:
  35       - 9081:9081
  36     stop_grace_period: 120s
  37     depends_on:
  38       - zookeeper
  39
  40   kafka-2:
  41     image: confluentinc/cp-kafka:7.5.1
  42     environment:
  43       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  44       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  45       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  46       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  47       KAFKA_BROKER_ID: 2
  48       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  49       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  50       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  51       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  52       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  53       KAFKA_OPTS:
  54         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  55         -Dcom.sun.management.jmxremote=true
  56         -Dcom.sun.management.jmxremote.port=9101
  57         -Dcom.sun.management.jmxremote.authenticate=false
  58     volumes:
  59       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  60       - kafka-2-data:/var/lib/kafka/data
  61     ports:
  62       - 9092:9082
  63       - 9082:9082
  64     networks:
  65       default:
  66         aliases:
  67           - kafka
  68     stop_grace_period: 120s
  69     depends_on:
  70       - zookeeper
  71
  72   kafka-3:
  73     image: confluentinc/cp-kafka:7.5.1
  74     environment:
  75       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  76       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  77       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  78       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  79       KAFKA_BROKER_ID: 3
  80       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  81       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  82       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  83       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  84       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  85       KAFKA_OPTS:
  86         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  87         -Dcom.sun.management.jmxremote=true
  88         -Dcom.sun.management.jmxremote.port=9101
  89         -Dcom.sun.management.jmxremote.authenticate=false
  90     volumes:
  91       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  92       - kafka-3-data:/var/lib/kafka/data
  93     ports:
  94       - 9083:9083
  95     stop_grace_period: 120s
  96     depends_on:
  97       - zookeeper
  98
  99   schema-registry:
 100     image: confluentinc/cp-schema-registry:7.5.1
 101     environment:
 102       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 103       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 104       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 105       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 106     ports:
 107       - 8085:8085
 108     depends_on:
 109       - kafka-1
 110       - kafka-2
 111       - kafka-3
 112
 113   connect:
 114     image: confluentinc/cp-kafka-connect:7.5.1
 115     environment:
 116       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 117       CONNECT_REST_PORT: 8083
 118       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 119       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 120       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 121       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 122       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 123       CONNECT_GROUP_ID: kafka-connect
 124       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 125       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 126       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 127       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 128       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 129       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 130       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 131       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 132       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 133       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 134       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 135       CONNECT_PLUGIN_PATH: /usr/share/java/
 136     ports:
 137       - 8083:8083
 138     depends_on:
 139       - schema-registry
 140
 141   cli:
 142     image: juplo/toolbox
 143     command:
 144       - bash
 145       - -c
 146       - >-
 147         cd /home/appuser;
 148         cat movies.json | kafka-avro-console-producer
 149         --bootstrap-server kafka:9092
 150         --topic test
 151         --property schema.registry.url=http://schema-registry:8085
 152         --property value.schema="$$(cat movie.avsc)";
 153         exec sleep infinity
 154     volumes:
 155       - ./movie.avsc:/home/appuser/movie.avsc:ro
 156       - ./movies.json:/home/appuser/movies.json:ro
 157     stop_grace_period: 0s
 158     depends_on:
 159       - kafka-1
 160       - kafka-2
 161       - kafka-3
 162
 163   setup:
 164     image: juplo/toolbox
 165     command:
 166       - bash
 167       - -c
 168       - |
 169         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 170         if [ -e INITIALIZED ]
 171         then
 172           echo -n Bereits konfiguriert:
 173           cat INITIALIZED
 174           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 175         else
 176           kafka-topics --bootstrap-server kafka:9092 \
 177                        --delete \
 178                        --if-exists \
 179                        --topic test
 180           kafka-topics --bootstrap-server kafka:9092 \
 181                        --create \
 182                        --topic test \
 183                        --partitions 2 \
 184                        --replication-factor 3 \
 185                        --config min.insync.replicas=2 \
 186           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 187           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
 188           && date > INITIALIZED
 189         fi
 190     stop_grace_period: 0s
 191     depends_on:
 192       - cli
 193
 194   zoonavigator:
 195     image: elkozmon/zoonavigator:1.1.2
 196     ports:
 197       - "8000:80"
 198     environment:
 199       HTTP_PORT: 80
 200       CONNECTION_JUPLO_NAME: juplo
 201       CONNECTION_JUPLO_CONN: zookeeper:2181
 202       AUTO_CONNECT_CONNECTION_ID: JUPLO
 203     depends_on:
 204       - zookeeper
 205
 206   kafdrop:
 207     image: obsidiandynamics/kafdrop:3.31.0
 208     ports:
 209       - 8888:9000
 210     environment:
 211       KAFKA_BROKERCONNECT: kafka-1:9092,kafka-2:9092,kafka-3:9092
 212       JVM_OPTS: -Xms32M -Xmx64M
 213       SERVER_SERVLET_CONTEXTPATH: /
 214       CMD_ARGS: --schemaregistry.connect=http://schema-registry:8085
 215     depends_on:
 216       - kafka-1
 217       - kafka-2
 218       - kafka-3
 219
 220 volumes:
 221   zookeeper-data:
 222   zookeeper-log:
 223   kafka-1-data:
 224   kafka-2-data:
 225   kafka-3-data: