juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
  26       KAFKA_SUPER_USERS: User:ANONYMOUS
  27       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
  28       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  29       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  30       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  31       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  32         org.apache.kafka.common.security.plain.PlainLoginModule required \
  33         user_schemaregistry="schemaregistry-secret" \
  34         user_connect="connect-secret" \
  35         user_ui="ui-secret" \
  36         user_client="client-secret";
  37       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  38       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  39         org.apache.kafka.common.security.plain.PlainLoginModule required \
  40         user_client="client-secret";
  41       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  42     volumes:
  43       - kafka-1-data:/var/lib/kafka/data
  44     ports:
  45       - 9081:9081
  46     stop_grace_period: 120s
  47     depends_on:
  48       - zookeeper
  49
  50   kafka-2:
  51     image: juplo/kafka
  52     environment:
  53       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  54       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  55       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  56       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  57       KAFKA_BROKER_ID: 2
  58       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  59       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  60       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  61       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  62       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
  63       KAFKA_SUPER_USERS: User:ANONYMOUS
  64       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
  65       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  66       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  67       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  68       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  69         org.apache.kafka.common.security.plain.PlainLoginModule required \
  70         user_schemaregistry="schemaregistry-secret" \
  71         user_connect="connect-secret" \
  72         user_ui="ui-secret" \
  73         user_client="client-secret";
  74       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  75       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  76         org.apache.kafka.common.security.plain.PlainLoginModule required \
  77         user_client="client-secret";
  78       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  79     volumes:
  80       - kafka-2-data:/var/lib/kafka/data
  81     ports:
  82       - 9092:9082
  83       - 9082:9082
  84     networks:
  85       default:
  86         aliases:
  87           - kafka
  88     stop_grace_period: 120s
  89     depends_on:
  90       - zookeeper
  91
  92   kafka-3:
  93     image: juplo/kafka
  94     environment:
  95       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  96       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
  97       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  98       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
  99       KAFKA_BROKER_ID: 3
 100       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 101       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 102       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 103       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 104       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
 105       KAFKA_SUPER_USERS: User:ANONYMOUS
 106       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
 107       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 108       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 109       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 110       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 111         org.apache.kafka.common.security.plain.PlainLoginModule required \
 112         user_schemaregistry="schemaregistry-secret" \
 113         user_connect="connect-secret" \
 114         user_ui="ui-secret" \
 115         user_client="client-secret";
 116       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 117       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 118         org.apache.kafka.common.security.plain.PlainLoginModule required \
 119         user_client="client-secret";
 120       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 121     volumes:
 122       - kafka-3-data:/var/lib/kafka/data
 123     ports:
 124       - 9083:9083
 125     stop_grace_period: 120s
 126     depends_on:
 127       - zookeeper
 128
 129   kafka-4:
 130     image: juplo/kafka
 131     environment:
 132       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 133       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 134       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 135       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 136       KAFKA_BROKER_ID: 4
 137       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 138       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 139       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 140       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 141       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
 142       KAFKA_SUPER_USERS: User:ANONYMOUS
 143       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
 144       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 145       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 146       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 147       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 148         org.apache.kafka.common.security.plain.PlainLoginModule required \
 149         user_schemaregistry="schemaregistry-secret" \
 150         user_connect="connect-secret" \
 151         user_ui="ui-secret" \
 152         user_client="client-secret";
 153       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 154       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 155         org.apache.kafka.common.security.plain.PlainLoginModule required \
 156         user_client="client-secret";
 157       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 158     volumes:
 159       - kafka-4-data:/var/lib/kafka/data
 160     ports:
 161       - 9084:9084
 162     stop_grace_period: 120s
 163     depends_on:
 164       - zookeeper
 165
 166   cruise-control:
 167     image: juplo/cruise-control
 168     environment:
 169       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 170       ZOOKEEPER_CONNECT: zookeeper:2181/
 171     ports:
 172       - "9090:9090"
 173
 174   schema-registry:
 175     image: confluentinc/cp-schema-registry:7.5.1
 176     environment:
 177       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 178       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 179       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 180       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 181       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 182       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 183       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 184         org.apache.kafka.common.security.plain.PlainLoginModule
 185         required
 186         username="schemaregistry"
 187         password="schemaregistry-secret";
 188     ports:
 189       - 8085:8085
 190     depends_on:
 191       - kafka-1
 192       - kafka-2
 193       - kafka-3
 194
 195   connect:
 196     image: confluentinc/cp-kafka-connect:7.5.1
 197     environment:
 198       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 199       CONNECT_REST_PORT: 8083
 200       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 201       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 202       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 203       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 204       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 205       CONNECT_GROUP_ID: kafka-connect
 206       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 207       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 208       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 209       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 210       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 211       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 212       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 213       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 214       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 215       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 216       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 217       CONNECT_PLUGIN_PATH: /usr/share/java/
 218       # Configure the Connect workers to use SASL/PLAIN.
 219       CONNECT_SASL_MECHANISM: PLAIN
 220       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 221       # JAAS
 222       CONNECT_SASL_JAAS_CONFIG: >
 223         org.apache.kafka.common.security.plain.PlainLoginModule required
 224         username="connect"
 225         password="connect-secret";
 226       # producer
 227       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 228       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 229       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 230         org.apache.kafka.common.security.plain.PlainLoginModule required
 231         username="connect"
 232         password="connect-secret";
 233       # consumer
 234       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 235       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 236       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 237         org.apache.kafka.common.security.plain.PlainLoginModule required
 238         username="connect"
 239         password="connect-secret";
 240
 241     ports:
 242       - 8083:8083
 243     depends_on:
 244       - schema-registry
 245
 246   cli:
 247     image: juplo/toolbox
 248     command: sleep infinity
 249     stop_grace_period: 0s
 250     depends_on:
 251       - kafka-1
 252       - kafka-2
 253       - kafka-3
 254
 255   setup:
 256     image: juplo/toolbox
 257     command:
 258       - bash
 259       - -c
 260       - |
 261         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 262         if [ -e INITIALIZED ]
 263         then
 264           echo -n Bereits konfiguriert:
 265           cat INITIALIZED
 266           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 267           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 268         else
 269           kafka-topics --bootstrap-server kafka:9091 \
 270                        --delete \
 271                        --if-exists \
 272                        --topic test
 273           kafka-topics --bootstrap-server kafka:9091 \
 274                        --create \
 275                        --topic test \
 276                        --partitions 2 \
 277                        --replication-factor 3 \
 278                        --config min.insync.replicas=2 \
 279           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 280           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 281           kafka-topics --bootstrap-server kafka:9091 \
 282                        --delete \
 283                        --if-exists \
 284                        --topic __CruiseControlMetrics
 285           kafka-topics --bootstrap-server kafka:9091 \
 286                        --create \
 287                        --topic __CruiseControlMetrics \
 288                        --partitions 2 \
 289                        --replication-factor 3 \
 290                        --config min.insync.replicas=2 \
 291           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 292           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 293         fi \
 294         && date > INITIALIZED
 295     stop_grace_period: 0s
 296     depends_on:
 297       - cli
 298
 299   zoonavigator:
 300     image: elkozmon/zoonavigator:1.1.2
 301     ports:
 302       - "8000:80"
 303     environment:
 304       HTTP_PORT: 80
 305       CONNECTION_JUPLO_NAME: juplo
 306       CONNECTION_JUPLO_CONN: zookeeper:2181
 307       AUTO_CONNECT_CONNECTION_ID: JUPLO
 308     depends_on:
 309       - zookeeper
 310
 311   akhq:
 312     image: tchiotludo/akhq:0.23.0
 313     ports:
 314       - 8888:8080
 315     environment:
 316       AKHQ_CONFIGURATION: |
 317         akhq:
 318           connections:
 319             docker-kafka-server:
 320               properties:
 321                 bootstrap.servers: "kafka:9092"
 322                 security.protocol: SASL_PLAINTEXT
 323                 sasl.mechanism: PLAIN
 324                 sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="ui" password="ui-secret";
 325               schema-registry:
 326                 url: "http://schema-registry:8085"
 327               connect:
 328                 - name: "connect"
 329                   url: "http://connect:8083"
 330     depends_on:
 331       - kafka-1
 332       - kafka-2
 333       - kafka-3
 334
 335   producer:
 336     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 337     environment:
 338       producer.throttle-ms: 1
 339       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 340       spring.kafka.client-id: producer
 341       spring.kafka.template.default-topic: test
 342       spring.kafka.producer.properties.linger.ms: 100
 343       spring.kafka.jaas.enabled: "true"
 344       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 345       spring.kafka.properties.sasl.mechanism: PLAIN
 346       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 347
 348   consumer-1:
 349     image: juplo/spring-consumer:1.0-SNAPSHOT
 350     environment:
 351       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 352       spring.kafka.client-id: consumer-1
 353       spring.kafka.jaas.enabled: "true"
 354       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 355       spring.kafka.properties.sasl.mechanism: PLAIN
 356       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 357
 358   consumer-2:
 359     image: juplo/spring-consumer:1.0-SNAPSHOT
 360     environment:
 361       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 362       spring.kafka.client-id: consumer-2
 363       spring.kafka.jaas.enabled: "true"
 364       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 365       spring.kafka.properties.sasl.mechanism: PLAIN
 366       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 367
 368   consumer-3:
 369     image: juplo/spring-consumer:1.0-SNAPSHOT
 370     environment:
 371       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 372       spring.kafka.client-id: consumer-3
 373       spring.kafka.jaas.enabled: "true"
 374       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 375       spring.kafka.properties.sasl.mechanism: PLAIN
 376       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
 377
 378 volumes:
 379   zookeeper-data:
 380   zookeeper-log:
 381   kafka-1-data:
 382   kafka-2-data:
 383   kafka-3-data:
 384   kafka-4-data: