juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  32       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  33       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  34       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  35         org.apache.kafka.common.security.plain.PlainLoginModule required \
  36         user_schemaregistry="schemaregistry-secret" \
  37         user_connect="connect-secret" \
  38         user_ui="ui-secret" \
  39         user_client="client-secret";
  40       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  41       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  42         org.apache.kafka.common.security.plain.PlainLoginModule required \
  43         user_client="client-secret";
  44       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  45       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  46     volumes:
  47       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  48       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  49       - kafka-1-data:/var/lib/kafka/data
  50     ports:
  51       - 9081:9081
  52     stop_grace_period: 120s
  53     depends_on:
  54       - zookeeper
  55
  56   kafka-2:
  57     image: confluentinc/cp-kafka:7.5.1
  58     environment:
  59       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  60       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  61       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  62       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  63       KAFKA_BROKER_ID: 2
  64       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  65       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  66       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  67       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  68       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  69       KAFKA_OPTS:
  70         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  71         -Dcom.sun.management.jmxremote=true
  72         -Dcom.sun.management.jmxremote.port=9101
  73         -Dcom.sun.management.jmxremote.authenticate=false
  74       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  75       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  76       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  77       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  78         org.apache.kafka.common.security.plain.PlainLoginModule required \
  79         user_schemaregistry="schemaregistry-secret" \
  80         user_connect="connect-secret" \
  81         user_ui="ui-secret" \
  82         user_client="client-secret";
  83       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  84       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  85         org.apache.kafka.common.security.plain.PlainLoginModule required \
  86         user_client="client-secret";
  87       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  88       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  89     volumes:
  90       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  91       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  92       - kafka-2-data:/var/lib/kafka/data
  93     ports:
  94       - 9092:9082
  95       - 9082:9082
  96     networks:
  97       default:
  98         aliases:
  99           - kafka
 100     stop_grace_period: 120s
 101     depends_on:
 102       - zookeeper
 103
 104   kafka-3:
 105     image: confluentinc/cp-kafka:7.5.1
 106     environment:
 107       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 108       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
 109       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 110       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
 111       KAFKA_BROKER_ID: 3
 112       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 113       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 114       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 115       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 116       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 117       KAFKA_OPTS:
 118         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 119         -Dcom.sun.management.jmxremote=true
 120         -Dcom.sun.management.jmxremote.port=9101
 121         -Dcom.sun.management.jmxremote.authenticate=false
 122       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 123       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 124       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 125       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 126         org.apache.kafka.common.security.plain.PlainLoginModule required \
 127         user_schemaregistry="schemaregistry-secret" \
 128         user_connect="connect-secret" \
 129         user_ui="ui-secret" \
 130         user_client="client-secret";
 131       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 132       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 133         org.apache.kafka.common.security.plain.PlainLoginModule required \
 134         user_client="client-secret";
 135       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 136       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 137     volumes:
 138       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 139       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 140       - kafka-3-data:/var/lib/kafka/data
 141     ports:
 142       - 9083:9083
 143     stop_grace_period: 120s
 144     depends_on:
 145       - zookeeper
 146
 147   kafka-4:
 148     image: confluentinc/cp-kafka:7.5.1
 149     environment:
 150       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 151       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 152       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 153       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 154       KAFKA_BROKER_ID: 4
 155       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 156       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 157       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 158       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 159       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 160       KAFKA_OPTS:
 161         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 162         -Dcom.sun.management.jmxremote=true
 163         -Dcom.sun.management.jmxremote.port=9101
 164         -Dcom.sun.management.jmxremote.authenticate=false
 165       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 166       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 167       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 168       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 169         org.apache.kafka.common.security.plain.PlainLoginModule required \
 170         user_schemaregistry="schemaregistry-secret" \
 171         user_connect="connect-secret" \
 172         user_ui="ui-secret" \
 173         user_client="client-secret";
 174       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 175       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 176         org.apache.kafka.common.security.plain.PlainLoginModule required \
 177         user_client="client-secret";
 178       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 179       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 180     volumes:
 181       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 182       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 183       - kafka-4-data:/var/lib/kafka/data
 184     ports:
 185       - 9084:9084
 186     stop_grace_period: 120s
 187     depends_on:
 188       - zookeeper
 189
 190   cruise-control:
 191     image: juplo/cruise-control
 192     environment:
 193       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 194       ZOOKEEPER_CONNECT: zookeeper:2181/
 195     ports:
 196       - "9090:9090"
 197
 198   schema-registry:
 199     image: confluentinc/cp-schema-registry:7.5.1
 200     environment:
 201       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 202       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 203       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 204       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 205       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 206       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 207       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 208         org.apache.kafka.common.security.plain.PlainLoginModule
 209         required
 210         username="schemaregistry"
 211         password="schemaregistry-secret";
 212     ports:
 213       - 8085:8085
 214     depends_on:
 215       - kafka-1
 216       - kafka-2
 217       - kafka-3
 218
 219   connect:
 220     image: confluentinc/cp-kafka-connect:7.5.1
 221     environment:
 222       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 223       CONNECT_REST_PORT: 8083
 224       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 225       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 226       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 227       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 228       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 229       CONNECT_GROUP_ID: kafka-connect
 230       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 231       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 232       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 233       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 234       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 235       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 236       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 237       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 238       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 239       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 240       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 241       CONNECT_PLUGIN_PATH: /usr/share/java/
 242       # Configure the Connect workers to use SASL/PLAIN.
 243       CONNECT_SASL_MECHANISM: PLAIN
 244       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 245       # JAAS
 246       CONNECT_SASL_JAAS_CONFIG: >
 247         org.apache.kafka.common.security.plain.PlainLoginModule required
 248         username="connect"
 249         password="connect-secret";
 250       # producer
 251       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 252       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 253       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 254         org.apache.kafka.common.security.plain.PlainLoginModule required
 255         username="connect"
 256         password="connect-secret";
 257       # consumer
 258       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 259       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 260       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 261         org.apache.kafka.common.security.plain.PlainLoginModule required
 262         username="connect"
 263         password="connect-secret";
 264
 265     ports:
 266       - 8083:8083
 267     depends_on:
 268       - schema-registry
 269
 270   cli:
 271     image: juplo/toolbox
 272     command: sleep infinity
 273     stop_grace_period: 0s
 274     depends_on:
 275       - kafka-1
 276       - kafka-2
 277       - kafka-3
 278
 279   setup:
 280     image: juplo/toolbox
 281     command:
 282       - bash
 283       - -c
 284       - |
 285         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 286         if [ -e INITIALIZED ]
 287         then
 288           echo -n Bereits konfiguriert:
 289           cat INITIALIZED
 290           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 291           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 292         else
 293           kafka-topics --bootstrap-server kafka:9091 \
 294                        --delete \
 295                        --if-exists \
 296                        --topic test
 297           kafka-topics --bootstrap-server kafka:9091 \
 298                        --create \
 299                        --topic test \
 300                        --partitions 2 \
 301                        --replication-factor 3 \
 302                        --config min.insync.replicas=2 \
 303           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 304           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 305           kafka-topics --bootstrap-server kafka:9091 \
 306                        --delete \
 307                        --if-exists \
 308                        --topic __CruiseControlMetrics
 309           kafka-topics --bootstrap-server kafka:9091 \
 310                        --create \
 311                        --topic __CruiseControlMetrics \
 312                        --partitions 2 \
 313                        --replication-factor 3 \
 314                        --config min.insync.replicas=2 \
 315           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 316           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 317         fi \
 318         && date > INITIALIZED
 319     stop_grace_period: 0s
 320     depends_on:
 321       - cli
 322
 323   zoonavigator:
 324     image: elkozmon/zoonavigator:1.1.2
 325     ports:
 326       - "8000:80"
 327     environment:
 328       HTTP_PORT: 80
 329       CONNECTION_JUPLO_NAME: juplo
 330       CONNECTION_JUPLO_CONN: zookeeper:2181
 331       AUTO_CONNECT_CONNECTION_ID: JUPLO
 332     depends_on:
 333       - zookeeper
 334
 335   akhq:
 336     image: tchiotludo/akhq:0.23.0
 337     ports:
 338       - 8888:8080
 339     environment:
 340       AKHQ_CONFIGURATION: |
 341         akhq:
 342           connections:
 343             docker-kafka-server:
 344               properties:
 345                 bootstrap.servers: "kafka:9092"
 346                 security.protocol: SASL_PLAINTEXT
 347                 sasl.mechanism: PLAIN
 348                 sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="ui" password="ui-secret";
 349               schema-registry:
 350                 url: "http://schema-registry:8085"
 351               connect:
 352                 - name: "connect"
 353                   url: "http://connect:8083"
 354     depends_on:
 355       - kafka-1
 356       - kafka-2
 357       - kafka-3
 358
 359   producer:
 360     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 361     environment:
 362       producer.throttle-ms: 1
 363       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 364       spring.kafka.client-id: producer
 365       spring.kafka.template.default-topic: test
 366       spring.kafka.producer.properties.linger.ms: 100
 367       spring.kafka.jaas.enabled: "true"
 368       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 369       spring.kafka.properties.sasl.mechanism: PLAIN
 370       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 371
 372   consumer-1:
 373     image: juplo/spring-consumer:1.0-SNAPSHOT
 374     environment:
 375       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 376       spring.kafka.client-id: consumer-1
 377       spring.kafka.jaas.enabled: "true"
 378       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 379       spring.kafka.properties.sasl.mechanism: PLAIN
 380       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 381
 382   consumer-2:
 383     image: juplo/spring-consumer:1.0-SNAPSHOT
 384     environment:
 385       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 386       spring.kafka.client-id: consumer-1
 387       spring.kafka.jaas.enabled: "true"
 388       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 389       spring.kafka.properties.sasl.mechanism: PLAIN
 390       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 391
 392   consumer-3:
 393     image: juplo/spring-consumer:1.0-SNAPSHOT
 394     environment:
 395       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 396       spring.kafka.client-id: consumer-1
 397       spring.kafka.jaas.enabled: "true"
 398       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 399       spring.kafka.properties.sasl.mechanism: PLAIN
 400       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 401
 402 volumes:
 403   zookeeper-data:
 404   zookeeper-log:
 405   kafka-1-data:
 406   kafka-2-data:
 407   kafka-3-data:
 408   kafka-4-data: