juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  32     volumes:
  33       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  34       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  35       - kafka-1-data:/var/lib/kafka/data
  36     ports:
  37       - 9081:9081
  38     stop_grace_period: 120s
  39     depends_on:
  40       - zookeeper
  41
  42   kafka-2:
  43     image: confluentinc/cp-kafka:7.5.1
  44     environment:
  45       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  46       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  47       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  48       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  49       KAFKA_BROKER_ID: 2
  50       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  51       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  52       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  53       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  54       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  55       KAFKA_OPTS:
  56         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  57         -Dcom.sun.management.jmxremote=true
  58         -Dcom.sun.management.jmxremote.port=9101
  59         -Dcom.sun.management.jmxremote.authenticate=false
  60       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  61     volumes:
  62       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  63       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  64       - kafka-2-data:/var/lib/kafka/data
  65     ports:
  66       - 9092:9082
  67       - 9082:9082
  68     networks:
  69       default:
  70         aliases:
  71           - kafka
  72     stop_grace_period: 120s
  73     depends_on:
  74       - zookeeper
  75
  76   kafka-3:
  77     image: confluentinc/cp-kafka:7.5.1
  78     environment:
  79       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  80       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  81       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  82       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  83       KAFKA_BROKER_ID: 3
  84       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  85       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  86       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  87       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  88       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  89       KAFKA_OPTS:
  90         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  91         -Dcom.sun.management.jmxremote=true
  92         -Dcom.sun.management.jmxremote.port=9101
  93         -Dcom.sun.management.jmxremote.authenticate=false
  94       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  95     volumes:
  96       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  97       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  98       - kafka-3-data:/var/lib/kafka/data
  99     ports:
 100       - 9083:9083
 101     stop_grace_period: 120s
 102     depends_on:
 103       - zookeeper
 104
 105   kafka-4:
 106     image: confluentinc/cp-kafka:7.5.1
 107     environment:
 108       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 109       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9084
 110       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
 111       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
 112       KAFKA_BROKER_ID: 4
 113       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 114       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 115       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 116       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 117       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 118       KAFKA_OPTS:
 119         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 120         -Dcom.sun.management.jmxremote=true
 121         -Dcom.sun.management.jmxremote.port=9101
 122         -Dcom.sun.management.jmxremote.authenticate=false
 123       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 124     volumes:
 125       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 126       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 127       - kafka-4-data:/var/lib/kafka/data
 128     ports:
 129       - 9084:9084
 130     stop_grace_period: 120s
 131     depends_on:
 132       - zookeeper
 133
 134   cruise-control:
 135     image: juplo/cruise-control
 136     environment:
 137       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 138       ZOOKEEPER_CONNECT: zookeeper:2181/
 139     ports:
 140       - "9090:9090"
 141
 142   schema-registry:
 143     image: confluentinc/cp-schema-registry:7.5.1
 144     environment:
 145       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 146       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 147       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 148       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 149     ports:
 150       - 8085:8085
 151     depends_on:
 152       - kafka-1
 153       - kafka-2
 154       - kafka-3
 155
 156   connect:
 157     image: confluentinc/cp-kafka-connect:7.5.1
 158     environment:
 159       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 160       CONNECT_REST_PORT: 8083
 161       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 162       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 163       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 164       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 165       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 166       CONNECT_GROUP_ID: kafka-connect
 167       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 168       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 169       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 170       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 171       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 172       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 173       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 174       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 175       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 176       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 177       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 178       CONNECT_PLUGIN_PATH: /usr/share/java/
 179     ports:
 180       - 8083:8083
 181     depends_on:
 182       - schema-registry
 183
 184   cli:
 185     image: juplo/toolbox
 186     command: sleep infinity
 187     stop_grace_period: 0s
 188     depends_on:
 189       - kafka-1
 190       - kafka-2
 191       - kafka-3
 192
 193   setup:
 194     image: juplo/toolbox
 195     command:
 196       - bash
 197       - -c
 198       - |
 199         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 200         if [ -e INITIALIZED ]
 201         then
 202           echo -n Bereits konfiguriert:
 203           cat INITIALIZED
 204           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 205           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 206         else
 207           kafka-topics --bootstrap-server kafka:9092 \
 208                        --delete \
 209                        --if-exists \
 210                        --topic test
 211           kafka-topics --bootstrap-server kafka:9092 \
 212                        --create \
 213                        --topic test \
 214                        --partitions 2 \
 215                        --replication-factor 3 \
 216                        --config min.insync.replicas=2 \
 217           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 218           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 219           kafka-topics --bootstrap-server kafka:9092 \
 220                        --delete \
 221                        --if-exists \
 222                        --topic __CruiseControlMetrics
 223           kafka-topics --bootstrap-server kafka:9092 \
 224                        --create \
 225                        --topic __CruiseControlMetrics \
 226                        --partitions 2 \
 227                        --replication-factor 3 \
 228                        --config min.insync.replicas=2 \
 229           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 230           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 231         fi \
 232         && date > INITIALIZED
 233     stop_grace_period: 0s
 234     depends_on:
 235       - cli
 236
 237   zoonavigator:
 238     image: elkozmon/zoonavigator:1.1.2
 239     ports:
 240       - "8000:80"
 241     environment:
 242       HTTP_PORT: 80
 243       CONNECTION_JUPLO_NAME: juplo
 244       CONNECTION_JUPLO_CONN: zookeeper:2181
 245       AUTO_CONNECT_CONNECTION_ID: JUPLO
 246     depends_on:
 247       - zookeeper
 248
 249   akhq:
 250     image: tchiotludo/akhq:0.23.0
 251     ports:
 252       - 8888:8080
 253     environment:
 254       AKHQ_CONFIGURATION: |
 255         akhq:
 256           connections:
 257             docker-kafka-server:
 258               properties:
 259                 bootstrap.servers: "kafka:9092"
 260               schema-registry:
 261                 url: "http://schema-registry:8085"
 262               connect:
 263                 - name: "connect"
 264                   url: "http://connect:8083"
 265     depends_on:
 266       - kafka-1
 267       - kafka-2
 268       - kafka-3
 269
 270   producer:
 271     image: juplo/simple-producer:1.0-SNAPSHOT
 272     command: kafka:9092 test producer
 273
 274   consumer-1:
 275     image: juplo/simple-consumer:1.0-SNAPSHOT
 276     command: kafka:9092 test my-group consumer
 277
 278   consumer-2:
 279     image: juplo/simple-consumer:1.0-SNAPSHOT
 280     command: kafka:9092 test my-group consumer-2
 281
 282   consumer-3:
 283     image: juplo/simple-consumer:1.0-SNAPSHOT
 284     command: kafka:9092 test my-group consumer-3
 285
 286 volumes:
 287   zookeeper-data:
 288   zookeeper-log:
 289   kafka-1-data:
 290   kafka-2-data:
 291   kafka-3-data:
 292   kafka-4-data: