juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  27       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  28       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  29       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  30         org.apache.kafka.common.security.plain.PlainLoginModule required \
  31         user_schemaregistry="schemaregistry-secret" \
  32         user_connect="connect-secret" \
  33         user_ui="ui-secret" \
  34         user_client="client-secret";
  35       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  36       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  37         org.apache.kafka.common.security.plain.PlainLoginModule required \
  38         user_client="client-secret";
  39       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  40     volumes:
  41       - kafka-1-data:/var/lib/kafka/data
  42     ports:
  43       - 9081:9081
  44     stop_grace_period: 120s
  45     depends_on:
  46       - zookeeper
  47
  48   kafka-2:
  49     image: juplo/kafka
  50     environment:
  51       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  52       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  53       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  54       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  55       KAFKA_BROKER_ID: 2
  56       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  57       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  58       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  59       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  60       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  61       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  62       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  63       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  64       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  65         org.apache.kafka.common.security.plain.PlainLoginModule required \
  66         user_schemaregistry="schemaregistry-secret" \
  67         user_connect="connect-secret" \
  68         user_ui="ui-secret" \
  69         user_client="client-secret";
  70       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  71       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  72         org.apache.kafka.common.security.plain.PlainLoginModule required \
  73         user_client="client-secret";
  74       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  75     volumes:
  76       - kafka-2-data:/var/lib/kafka/data
  77     ports:
  78       - 9092:9082
  79       - 9082:9082
  80     networks:
  81       default:
  82         aliases:
  83           - kafka
  84     stop_grace_period: 120s
  85     depends_on:
  86       - zookeeper
  87
  88   kafka-3:
  89     image: juplo/kafka
  90     environment:
  91       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  92       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
  93       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  94       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
  95       KAFKA_BROKER_ID: 3
  96       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  97       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  98       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  99       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 100       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 101       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 102       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 103       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 104       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 105         org.apache.kafka.common.security.plain.PlainLoginModule required \
 106         user_schemaregistry="schemaregistry-secret" \
 107         user_connect="connect-secret" \
 108         user_ui="ui-secret" \
 109         user_client="client-secret";
 110       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 111       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 112         org.apache.kafka.common.security.plain.PlainLoginModule required \
 113         user_client="client-secret";
 114       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 115     volumes:
 116       - kafka-3-data:/var/lib/kafka/data
 117     ports:
 118       - 9083:9083
 119     stop_grace_period: 120s
 120     depends_on:
 121       - zookeeper
 122
 123   kafka-4:
 124     image: juplo/kafka
 125     environment:
 126       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 127       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 128       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 129       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 130       KAFKA_BROKER_ID: 4
 131       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 132       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 133       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 134       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 135       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 136       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 137       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 138       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 139       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 140         org.apache.kafka.common.security.plain.PlainLoginModule required \
 141         user_schemaregistry="schemaregistry-secret" \
 142         user_connect="connect-secret" \
 143         user_ui="ui-secret" \
 144         user_client="client-secret";
 145       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 146       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 147         org.apache.kafka.common.security.plain.PlainLoginModule required \
 148         user_client="client-secret";
 149       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 150     volumes:
 151       - kafka-4-data:/var/lib/kafka/data
 152     ports:
 153       - 9084:9084
 154     stop_grace_period: 120s
 155     depends_on:
 156       - zookeeper
 157
 158   cruise-control:
 159     image: juplo/cruise-control
 160     environment:
 161       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 162       ZOOKEEPER_CONNECT: zookeeper:2181/
 163     ports:
 164       - "9090:9090"
 165
 166   schema-registry:
 167     image: confluentinc/cp-schema-registry:7.5.1
 168     environment:
 169       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 170       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 171       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 172       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 173       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 174       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 175       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 176         org.apache.kafka.common.security.plain.PlainLoginModule
 177         required
 178         username="schemaregistry"
 179         password="schemaregistry-secret";
 180     ports:
 181       - 8085:8085
 182     depends_on:
 183       - kafka-1
 184       - kafka-2
 185       - kafka-3
 186
 187   connect:
 188     image: confluentinc/cp-kafka-connect:7.5.1
 189     environment:
 190       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 191       CONNECT_REST_PORT: 8083
 192       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 193       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 194       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 195       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 196       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 197       CONNECT_GROUP_ID: kafka-connect
 198       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 199       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 200       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 201       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 202       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 203       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 204       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 205       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 206       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 207       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 208       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 209       CONNECT_PLUGIN_PATH: /usr/share/java/
 210       # Configure the Connect workers to use SASL/PLAIN.
 211       CONNECT_SASL_MECHANISM: PLAIN
 212       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 213       # JAAS
 214       CONNECT_SASL_JAAS_CONFIG: >
 215         org.apache.kafka.common.security.plain.PlainLoginModule required
 216         username="connect"
 217         password="connect-secret";
 218       # producer
 219       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 220       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 221       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 222         org.apache.kafka.common.security.plain.PlainLoginModule required
 223         username="connect"
 224         password="connect-secret";
 225       # consumer
 226       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 227       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 228       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 229         org.apache.kafka.common.security.plain.PlainLoginModule required
 230         username="connect"
 231         password="connect-secret";
 232
 233     ports:
 234       - 8083:8083
 235     depends_on:
 236       - schema-registry
 237
 238   cli:
 239     image: juplo/toolbox
 240     command: sleep infinity
 241     stop_grace_period: 0s
 242     depends_on:
 243       - kafka-1
 244       - kafka-2
 245       - kafka-3
 246
 247   setup:
 248     image: juplo/toolbox
 249     command:
 250       - bash
 251       - -c
 252       - |
 253         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 254         if [ -e INITIALIZED ]
 255         then
 256           echo -n Bereits konfiguriert:
 257           cat INITIALIZED
 258           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 259           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 260         else
 261           kafka-topics --bootstrap-server kafka:9091 \
 262                        --delete \
 263                        --if-exists \
 264                        --topic test
 265           kafka-topics --bootstrap-server kafka:9091 \
 266                        --create \
 267                        --topic test \
 268                        --partitions 2 \
 269                        --replication-factor 3 \
 270                        --config min.insync.replicas=2 \
 271           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 272           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 273           kafka-topics --bootstrap-server kafka:9091 \
 274                        --delete \
 275                        --if-exists \
 276                        --topic __CruiseControlMetrics
 277           kafka-topics --bootstrap-server kafka:9091 \
 278                        --create \
 279                        --topic __CruiseControlMetrics \
 280                        --partitions 2 \
 281                        --replication-factor 3 \
 282                        --config min.insync.replicas=2 \
 283           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 284           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 285         fi \
 286         && date > INITIALIZED
 287     stop_grace_period: 0s
 288     depends_on:
 289       - cli
 290
 291   zoonavigator:
 292     image: elkozmon/zoonavigator:1.1.2
 293     ports:
 294       - "8000:80"
 295     environment:
 296       HTTP_PORT: 80
 297       CONNECTION_JUPLO_NAME: juplo
 298       CONNECTION_JUPLO_CONN: zookeeper:2181
 299       AUTO_CONNECT_CONNECTION_ID: JUPLO
 300     depends_on:
 301       - zookeeper
 302
 303   akhq:
 304     image: tchiotludo/akhq:0.23.0
 305     ports:
 306       - 8888:8080
 307     environment:
 308       AKHQ_CONFIGURATION: |
 309         akhq:
 310           connections:
 311             docker-kafka-server:
 312               properties:
 313                 bootstrap.servers: "kafka:9092"
 314                 security.protocol: SASL_PLAINTEXT
 315                 sasl.mechanism: PLAIN
 316                 sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="ui" password="ui-secret";
 317               schema-registry:
 318                 url: "http://schema-registry:8085"
 319               connect:
 320                 - name: "connect"
 321                   url: "http://connect:8083"
 322     depends_on:
 323       - kafka-1
 324       - kafka-2
 325       - kafka-3
 326
 327   producer:
 328     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 329     environment:
 330       producer.throttle-ms: 1
 331       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 332       spring.kafka.client-id: producer
 333       spring.kafka.template.default-topic: test
 334       spring.kafka.producer.properties.linger.ms: 100
 335
 336   consumer-1:
 337     image: juplo/spring-consumer:1.0-SNAPSHOT
 338     environment:
 339       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 340       spring.kafka.client-id: consumer-1
 341
 342   consumer-2:
 343     image: juplo/spring-consumer:1.0-SNAPSHOT
 344     environment:
 345       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 346       spring.kafka.client-id: consumer-2
 347
 348   consumer-3:
 349     image: juplo/spring-consumer:1.0-SNAPSHOT
 350     environment:
 351       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 352       spring.kafka.client-id: consumer-3
 353
 354 volumes:
 355   zookeeper-data:
 356   zookeeper-log:
 357   kafka-1-data:
 358   kafka-2-data:
 359   kafka-3-data:
 360   kafka-4-data: