juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  32       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  33     volumes:
  34       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  35       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  36       - kafka-1-data:/var/lib/kafka/data
  37     ports:
  38       - 9081:9081
  39     stop_grace_period: 120s
  40     depends_on:
  41       - zookeeper
  42
  43   kafka-2:
  44     image: confluentinc/cp-kafka:7.5.1
  45     environment:
  46       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  47       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  48       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  49       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  50       KAFKA_BROKER_ID: 2
  51       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  52       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  53       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  54       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  55       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  56       KAFKA_OPTS:
  57         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  58         -Dcom.sun.management.jmxremote=true
  59         -Dcom.sun.management.jmxremote.port=9101
  60         -Dcom.sun.management.jmxremote.authenticate=false
  61       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  62       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  63     volumes:
  64       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  65       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  66       - kafka-2-data:/var/lib/kafka/data
  67     ports:
  68       - 9092:9082
  69       - 9082:9082
  70     networks:
  71       default:
  72         aliases:
  73           - kafka
  74     stop_grace_period: 120s
  75     depends_on:
  76       - zookeeper
  77
  78   kafka-3:
  79     image: confluentinc/cp-kafka:7.5.1
  80     environment:
  81       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  82       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
  83       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  84       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
  85       KAFKA_BROKER_ID: 3
  86       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  87       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  88       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  89       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  90       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  91       KAFKA_OPTS:
  92         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  93         -Dcom.sun.management.jmxremote=true
  94         -Dcom.sun.management.jmxremote.port=9101
  95         -Dcom.sun.management.jmxremote.authenticate=false
  96       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  97       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  98     volumes:
  99       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 100       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 101       - kafka-3-data:/var/lib/kafka/data
 102     ports:
 103       - 9083:9083
 104     stop_grace_period: 120s
 105     depends_on:
 106       - zookeeper
 107
 108   kafka-4:
 109     image: confluentinc/cp-kafka:7.5.1
 110     environment:
 111       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 112       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 113       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
 114       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 115       KAFKA_BROKER_ID: 4
 116       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 117       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 118       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 119       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 120       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 121       KAFKA_OPTS:
 122         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 123         -Dcom.sun.management.jmxremote=true
 124         -Dcom.sun.management.jmxremote.port=9101
 125         -Dcom.sun.management.jmxremote.authenticate=false
 126       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 127       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
 128     volumes:
 129       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 130       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 131       - kafka-4-data:/var/lib/kafka/data
 132     ports:
 133       - 9084:9084
 134     stop_grace_period: 120s
 135     depends_on:
 136       - zookeeper
 137
 138   cruise-control:
 139     image: juplo/cruise-control
 140     environment:
 141       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 142       ZOOKEEPER_CONNECT: zookeeper:2181/
 143     ports:
 144       - "9090:9090"
 145
 146   schema-registry:
 147     image: confluentinc/cp-schema-registry:7.5.1
 148     environment:
 149       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 150       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 151       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 152       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 153     ports:
 154       - 8085:8085
 155     depends_on:
 156       - kafka-1
 157       - kafka-2
 158       - kafka-3
 159
 160   connect:
 161     image: confluentinc/cp-kafka-connect:7.5.1
 162     environment:
 163       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 164       CONNECT_REST_PORT: 8083
 165       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 166       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 167       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 168       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 169       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 170       CONNECT_GROUP_ID: kafka-connect
 171       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 172       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 173       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 174       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 175       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 176       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 177       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 178       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 179       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 180       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 181       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 182       CONNECT_PLUGIN_PATH: /usr/share/java/
 183     ports:
 184       - 8083:8083
 185     depends_on:
 186       - schema-registry
 187
 188   cli:
 189     image: juplo/toolbox
 190     command: sleep infinity
 191     stop_grace_period: 0s
 192     depends_on:
 193       - kafka-1
 194       - kafka-2
 195       - kafka-3
 196
 197   setup:
 198     image: juplo/toolbox
 199     command:
 200       - bash
 201       - -c
 202       - |
 203         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 204         if [ -e INITIALIZED ]
 205         then
 206           echo -n Bereits konfiguriert:
 207           cat INITIALIZED
 208           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 209           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 210         else
 211           kafka-topics --bootstrap-server kafka:9091 \
 212                        --delete \
 213                        --if-exists \
 214                        --topic test
 215           kafka-topics --bootstrap-server kafka:9091 \
 216                        --create \
 217                        --topic test \
 218                        --partitions 2 \
 219                        --replication-factor 3 \
 220                        --config min.insync.replicas=2 \
 221           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 222           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 223           kafka-topics --bootstrap-server kafka:9091 \
 224                        --delete \
 225                        --if-exists \
 226                        --topic __CruiseControlMetrics
 227           kafka-topics --bootstrap-server kafka:9091 \
 228                        --create \
 229                        --topic __CruiseControlMetrics \
 230                        --partitions 2 \
 231                        --replication-factor 3 \
 232                        --config min.insync.replicas=2 \
 233           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 234           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 235         fi \
 236         && date > INITIALIZED
 237     stop_grace_period: 0s
 238     depends_on:
 239       - cli
 240
 241   zoonavigator:
 242     image: elkozmon/zoonavigator:1.1.2
 243     ports:
 244       - "8000:80"
 245     environment:
 246       HTTP_PORT: 80
 247       CONNECTION_JUPLO_NAME: juplo
 248       CONNECTION_JUPLO_CONN: zookeeper:2181
 249       AUTO_CONNECT_CONNECTION_ID: JUPLO
 250     depends_on:
 251       - zookeeper
 252
 253   akhq:
 254     image: tchiotludo/akhq:0.23.0
 255     ports:
 256       - 8888:8080
 257     environment:
 258       AKHQ_CONFIGURATION: |
 259         akhq:
 260           connections:
 261             docker-kafka-server:
 262               properties:
 263                 bootstrap.servers: "kafka:9091"
 264               schema-registry:
 265                 url: "http://schema-registry:8085"
 266               connect:
 267                 - name: "connect"
 268                   url: "http://connect:8083"
 269     depends_on:
 270       - kafka-1
 271       - kafka-2
 272       - kafka-3
 273
 274   producer:
 275     image: juplo/simple-producer:1.0-SNAPSHOT
 276     command: kafka:9092 test producer
 277
 278   consumer-1:
 279     image: juplo/spring-consumer:1.0-SNAPSHOT
 280     environment:
 281       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 282       spring.kafka.client-id: consumer-1
 283
 284   consumer-2:
 285     image: juplo/spring-consumer:1.0-SNAPSHOT
 286     environment:
 287       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 288       spring.kafka.client-id: consumer-1
 289
 290   consumer-3:
 291     image: juplo/spring-consumer:1.0-SNAPSHOT
 292     environment:
 293       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 294       spring.kafka.client-id: consumer-1
 295
 296 volumes:
 297   zookeeper-data:
 298   zookeeper-log:
 299   kafka-1-data:
 300   kafka-2-data:
 301   kafka-3-data:
 302   kafka-4-data: