juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  32       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  33       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  34       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  35         org.apache.kafka.common.security.plain.PlainLoginModule required \
  36         user_schemaregistry="schemaregistry-secret" \
  37         user_connect="connect-secret" \
  38         user_client="client-secret";
  39       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  40       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  41         org.apache.kafka.common.security.plain.PlainLoginModule required \
  42         user_client="client-secret";
  43       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  44       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  45     volumes:
  46       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  47       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  48       - kafka-1-data:/var/lib/kafka/data
  49     ports:
  50       - 9081:9081
  51     stop_grace_period: 120s
  52     depends_on:
  53       - zookeeper
  54
  55   kafka-2:
  56     image: confluentinc/cp-kafka:7.5.1
  57     environment:
  58       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  59       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  60       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  61       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  62       KAFKA_BROKER_ID: 2
  63       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  64       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  65       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  66       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  67       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  68       KAFKA_OPTS:
  69         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  70         -Dcom.sun.management.jmxremote=true
  71         -Dcom.sun.management.jmxremote.port=9101
  72         -Dcom.sun.management.jmxremote.authenticate=false
  73       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  74       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  75       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  76       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  77         org.apache.kafka.common.security.plain.PlainLoginModule required \
  78         user_schemaregistry="schemaregistry-secret" \
  79         user_connect="connect-secret" \
  80         user_client="client-secret";
  81       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  82       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  83         org.apache.kafka.common.security.plain.PlainLoginModule required \
  84         user_client="client-secret";
  85       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  86       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  87     volumes:
  88       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  89       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  90       - kafka-2-data:/var/lib/kafka/data
  91     ports:
  92       - 9092:9082
  93       - 9082:9082
  94     networks:
  95       default:
  96         aliases:
  97           - kafka
  98     stop_grace_period: 120s
  99     depends_on:
 100       - zookeeper
 101
 102   kafka-3:
 103     image: confluentinc/cp-kafka:7.5.1
 104     environment:
 105       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 106       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
 107       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 108       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
 109       KAFKA_BROKER_ID: 3
 110       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 111       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 112       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 113       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 114       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 115       KAFKA_OPTS:
 116         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 117         -Dcom.sun.management.jmxremote=true
 118         -Dcom.sun.management.jmxremote.port=9101
 119         -Dcom.sun.management.jmxremote.authenticate=false
 120       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 121       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 122       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 123       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 124         org.apache.kafka.common.security.plain.PlainLoginModule required \
 125         user_schemaregistry="schemaregistry-secret" \
 126         user_connect="connect-secret" \
 127         user_client="client-secret";
 128       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 129       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 130         org.apache.kafka.common.security.plain.PlainLoginModule required \
 131         user_client="client-secret";
 132       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 133       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 134     volumes:
 135       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 136       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 137       - kafka-3-data:/var/lib/kafka/data
 138     ports:
 139       - 9083:9083
 140     stop_grace_period: 120s
 141     depends_on:
 142       - zookeeper
 143
 144   kafka-4:
 145     image: confluentinc/cp-kafka:7.5.1
 146     environment:
 147       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 148       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 149       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 150       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 151       KAFKA_BROKER_ID: 4
 152       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 153       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 154       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 155       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 156       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 157       KAFKA_OPTS:
 158         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
 159         -Dcom.sun.management.jmxremote=true
 160         -Dcom.sun.management.jmxremote.port=9101
 161         -Dcom.sun.management.jmxremote.authenticate=false
 162       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 163       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 164       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 165       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 166         org.apache.kafka.common.security.plain.PlainLoginModule required \
 167         user_schemaregistry="schemaregistry-secret" \
 168         user_connect="connect-secret" \
 169         user_client="client-secret";
 170       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 171       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 172         org.apache.kafka.common.security.plain.PlainLoginModule required \
 173         user_client="client-secret";
 174       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 175       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 176     volumes:
 177       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
 178       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
 179       - kafka-4-data:/var/lib/kafka/data
 180     ports:
 181       - 9084:9084
 182     stop_grace_period: 120s
 183     depends_on:
 184       - zookeeper
 185
 186   cruise-control:
 187     image: juplo/cruise-control
 188     environment:
 189       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 190       ZOOKEEPER_CONNECT: zookeeper:2181/
 191     ports:
 192       - "9090:9090"
 193
 194   schema-registry:
 195     image: confluentinc/cp-schema-registry:7.5.1
 196     environment:
 197       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 198       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 199       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 200       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 201       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 202       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 203       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 204         org.apache.kafka.common.security.plain.PlainLoginModule
 205         required
 206         username="schemaregistry"
 207         password="schemaregistry-secret";
 208     ports:
 209       - 8085:8085
 210     depends_on:
 211       - kafka-1
 212       - kafka-2
 213       - kafka-3
 214
 215   connect:
 216     image: confluentinc/cp-kafka-connect:7.5.1
 217     environment:
 218       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 219       CONNECT_REST_PORT: 8083
 220       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 221       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 222       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 223       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 224       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 225       CONNECT_GROUP_ID: kafka-connect
 226       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 227       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 228       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 229       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 230       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 231       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 232       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 233       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 234       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 235       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 236       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 237       CONNECT_PLUGIN_PATH: /usr/share/java/
 238       # Configure the Connect workers to use SASL/PLAIN.
 239       CONNECT_SASL_MECHANISM: PLAIN
 240       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 241       # JAAS
 242       CONNECT_SASL_JAAS_CONFIG: >
 243         org.apache.kafka.common.security.plain.PlainLoginModule required
 244         username="connect"
 245         password="connect-secret";
 246       # producer
 247       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 248       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 249       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 250         org.apache.kafka.common.security.plain.PlainLoginModule required
 251         username="connect"
 252         password="connect-secret";
 253       # consumer
 254       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 255       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 256       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 257         org.apache.kafka.common.security.plain.PlainLoginModule required
 258         username="connect"
 259         password="connect-secret";
 260
 261     ports:
 262       - 8083:8083
 263     depends_on:
 264       - schema-registry
 265
 266   cli:
 267     image: juplo/toolbox
 268     command: sleep infinity
 269     stop_grace_period: 0s
 270     depends_on:
 271       - kafka-1
 272       - kafka-2
 273       - kafka-3
 274
 275   setup:
 276     image: juplo/toolbox
 277     command:
 278       - bash
 279       - -c
 280       - |
 281         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 282         if [ -e INITIALIZED ]
 283         then
 284           echo -n Bereits konfiguriert:
 285           cat INITIALIZED
 286           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 287           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 288         else
 289           kafka-topics --bootstrap-server kafka:9091 \
 290                        --delete \
 291                        --if-exists \
 292                        --topic test
 293           kafka-topics --bootstrap-server kafka:9091 \
 294                        --create \
 295                        --topic test \
 296                        --partitions 2 \
 297                        --replication-factor 3 \
 298                        --config min.insync.replicas=2 \
 299           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 300           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 301           kafka-topics --bootstrap-server kafka:9091 \
 302                        --delete \
 303                        --if-exists \
 304                        --topic __CruiseControlMetrics
 305           kafka-topics --bootstrap-server kafka:9091 \
 306                        --create \
 307                        --topic __CruiseControlMetrics \
 308                        --partitions 2 \
 309                        --replication-factor 3 \
 310                        --config min.insync.replicas=2 \
 311           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 312           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 313         fi \
 314         && date > INITIALIZED
 315     stop_grace_period: 0s
 316     depends_on:
 317       - cli
 318
 319   zoonavigator:
 320     image: elkozmon/zoonavigator:1.1.2
 321     ports:
 322       - "8000:80"
 323     environment:
 324       HTTP_PORT: 80
 325       CONNECTION_JUPLO_NAME: juplo
 326       CONNECTION_JUPLO_CONN: zookeeper:2181
 327       AUTO_CONNECT_CONNECTION_ID: JUPLO
 328     depends_on:
 329       - zookeeper
 330
 331   akhq:
 332     image: tchiotludo/akhq:0.23.0
 333     ports:
 334       - 8888:8080
 335     environment:
 336       AKHQ_CONFIGURATION: |
 337         akhq:
 338           connections:
 339             docker-kafka-server:
 340               properties:
 341                 bootstrap.servers: "kafka:9091"
 342               schema-registry:
 343                 url: "http://schema-registry:8085"
 344               connect:
 345                 - name: "connect"
 346                   url: "http://connect:8083"
 347     depends_on:
 348       - kafka-1
 349       - kafka-2
 350       - kafka-3
 351
 352   producer:
 353     image: juplo/simple-producer:1.0-SNAPSHOT
 354     command: kafka:9092 test producer
 355
 356   consumer-1:
 357     image: juplo/spring-consumer:1.0-SNAPSHOT
 358     environment:
 359       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 360       spring.kafka.client-id: consumer-1
 361
 362   consumer-2:
 363     image: juplo/spring-consumer:1.0-SNAPSHOT
 364     environment:
 365       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 366       spring.kafka.client-id: consumer-1
 367
 368   consumer-3:
 369     image: juplo/spring-consumer:1.0-SNAPSHOT
 370     environment:
 371       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 372       spring.kafka.client-id: consumer-1
 373
 374 volumes:
 375   zookeeper-data:
 376   zookeeper-log:
 377   kafka-1-data:
 378   kafka-2-data:
 379   kafka-3-data:
 380   kafka-4-data: