juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
  26       KAFKA_SUPER_USERS: User:ANONYMOUS
  27       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
  28       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  29       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  30       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  31       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  32       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  33       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  34       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  35     volumes:
  36       - kafka-1-data:/var/lib/kafka/data
  37     ports:
  38       - 9081:9081
  39     stop_grace_period: 120s
  40     depends_on:
  41       - zookeeper
  42
  43   kafka-2:
  44     image: juplo/kafka
  45     environment:
  46       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  47       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  48       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  49       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  50       KAFKA_BROKER_ID: 2
  51       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  52       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  53       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  54       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  55       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
  56       KAFKA_SUPER_USERS: User:ANONYMOUS
  57       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
  58       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  59       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  60       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  61       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  62       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  63       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  64       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  65     volumes:
  66       - kafka-2-data:/var/lib/kafka/data
  67     ports:
  68       - 9092:9082
  69       - 9082:9082
  70     networks:
  71       default:
  72         aliases:
  73           - kafka
  74     stop_grace_period: 120s
  75     depends_on:
  76       - zookeeper
  77
  78   kafka-3:
  79     image: juplo/kafka
  80     environment:
  81       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  82       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
  83       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  84       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
  85       KAFKA_BROKER_ID: 3
  86       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  87       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  88       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  89       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  90       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
  91       KAFKA_SUPER_USERS: User:ANONYMOUS
  92       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
  93       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  94       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  95       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  96       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  97       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
  98       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
  99       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 100     volumes:
 101       - kafka-3-data:/var/lib/kafka/data
 102     ports:
 103       - 9083:9083
 104     stop_grace_period: 120s
 105     depends_on:
 106       - zookeeper
 107
 108   kafka-4:
 109     image: juplo/kafka
 110     environment:
 111       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 112       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 113       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 114       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 115       KAFKA_BROKER_ID: 4
 116       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 117       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 118       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 119       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 120       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
 121       KAFKA_SUPER_USERS: User:ANONYMOUS
 122       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
 123       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 124       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 125       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
 126       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
 127       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
 128       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
 129       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 130     volumes:
 131       - kafka-4-data:/var/lib/kafka/data
 132     ports:
 133       - 9084:9084
 134     stop_grace_period: 120s
 135     depends_on:
 136       - zookeeper
 137
 138   cruise-control:
 139     image: juplo/cruise-control
 140     environment:
 141       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 142       ZOOKEEPER_CONNECT: zookeeper:2181/
 143     ports:
 144       - "9090:9090"
 145
 146   schema-registry:
 147     image: confluentinc/cp-schema-registry:7.5.1
 148     environment:
 149       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 150       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 151       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 152       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 153       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 154       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: SCRAM-SHA-512
 155       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 156         org.apache.kafka.common.security.scram.ScramLoginModule required
 157         username="schemaregistry"
 158         password="schemaregistry-secret";
 159     ports:
 160       - 8085:8085
 161     depends_on:
 162       - kafka-1
 163       - kafka-2
 164       - kafka-3
 165
 166   connect:
 167     image: confluentinc/cp-kafka-connect:7.5.1
 168     environment:
 169       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 170       CONNECT_REST_PORT: 8083
 171       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 172       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 173       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 174       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 175       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 176       CONNECT_GROUP_ID: kafka-connect
 177       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 178       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 179       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 180       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 181       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 182       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 183       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 184       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 185       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 186       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 187       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 188       CONNECT_PLUGIN_PATH: /usr/share/java/
 189       # Configure the Connect workers to use SASL/PLAIN.
 190       CONNECT_SASL_MECHANISM: SCRAM-SHA-512
 191       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 192       # JAAS
 193       CONNECT_SASL_JAAS_CONFIG: >
 194         org.apache.kafka.common.security.scram.ScramLoginModule required
 195         username="connect"
 196         password="connect-secret";
 197       # producer
 198       CONNECT_PRODUCER_SASL_MECHANISM: SCRAM-SHA-512
 199       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 200       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 201         org.apache.kafka.common.security.scram.ScramLoginModule required
 202         username="connect"
 203         password="connect-secret";
 204       # consumer
 205       CONNECT_CONSUMER_SASL_MECHANISM: SCRAM-SHA-512
 206       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 207       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 208         org.apache.kafka.common.security.scram.ScramLoginModule required
 209         username="connect"
 210         password="connect-secret";
 211
 212     ports:
 213       - 8083:8083
 214     depends_on:
 215       - schema-registry
 216
 217   cli:
 218     image: juplo/toolbox
 219     command: sleep infinity
 220     stop_grace_period: 0s
 221     depends_on:
 222       - kafka-1
 223       - kafka-2
 224       - kafka-3
 225
 226   setup:
 227     image: juplo/toolbox
 228     command:
 229       - bash
 230       - -c
 231       - |
 232         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 233         if [ -e INITIALIZED ]
 234         then
 235           echo -n Bereits konfiguriert:
 236           cat INITIALIZED
 237           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 238           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 239         else
 240           kafka-topics --bootstrap-server kafka:9091 \
 241                        --delete \
 242                        --if-exists \
 243                        --topic test
 244           kafka-topics --bootstrap-server kafka:9091 \
 245                        --create \
 246                        --topic test \
 247                        --partitions 2 \
 248                        --replication-factor 3 \
 249                        --config min.insync.replicas=2 \
 250           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 251           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 252           kafka-topics --bootstrap-server kafka:9091 \
 253                        --delete \
 254                        --if-exists \
 255                        --topic __CruiseControlMetrics
 256           kafka-topics --bootstrap-server kafka:9091 \
 257                        --create \
 258                        --topic __CruiseControlMetrics \
 259                        --partitions 2 \
 260                        --replication-factor 3 \
 261                        --config min.insync.replicas=2 \
 262           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 263           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 264         fi \
 265         && date > INITIALIZED
 266     stop_grace_period: 0s
 267     depends_on:
 268       - cli
 269
 270   zoonavigator:
 271     image: elkozmon/zoonavigator:1.1.2
 272     ports:
 273       - "8000:80"
 274     environment:
 275       HTTP_PORT: 80
 276       CONNECTION_JUPLO_NAME: juplo
 277       CONNECTION_JUPLO_CONN: zookeeper:2181
 278       AUTO_CONNECT_CONNECTION_ID: JUPLO
 279     depends_on:
 280       - zookeeper
 281
 282   akhq:
 283     image: tchiotludo/akhq:0.23.0
 284     ports:
 285       - 8888:8080
 286     environment:
 287       AKHQ_CONFIGURATION: |
 288         akhq:
 289           connections:
 290             docker-kafka-server:
 291               properties:
 292                 bootstrap.servers: "kafka:9092"
 293                 security.protocol: SASL_PLAINTEXT
 294                 sasl.mechanism: SCRAM-SHA-512
 295                 sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="ui" password="ui-secret";
 296               schema-registry:
 297                 url: "http://schema-registry:8085"
 298               connect:
 299                 - name: "connect"
 300                   url: "http://connect:8083"
 301     depends_on:
 302       - kafka-1
 303       - kafka-2
 304       - kafka-3
 305
 306   producer:
 307     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 308     environment:
 309       producer.throttle-ms: 1
 310       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 311       spring.kafka.client-id: producer
 312       spring.kafka.template.default-topic: test
 313       spring.kafka.producer.properties.linger.ms: 100
 314       spring.kafka.jaas.enabled: "true"
 315       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 316       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
 317       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
 318
 319   consumer-1:
 320     image: juplo/spring-consumer:1.0-SNAPSHOT
 321     environment:
 322       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 323       spring.kafka.client-id: consumer-1
 324       spring.kafka.jaas.enabled: "true"
 325       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 326       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
 327       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
 328
 329   consumer-2:
 330     image: juplo/spring-consumer:1.0-SNAPSHOT
 331     environment:
 332       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 333       spring.kafka.client-id: consumer-2
 334       spring.kafka.jaas.enabled: "true"
 335       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 336       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
 337       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
 338
 339   consumer-3:
 340     image: juplo/spring-consumer:1.0-SNAPSHOT
 341     environment:
 342       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 343       spring.kafka.client-id: consumer-3
 344       spring.kafka.jaas.enabled: "true"
 345       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
 346       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
 347       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
 348
 349 volumes:
 350   zookeeper-data:
 351   zookeeper-log:
 352   kafka-1-data:
 353   kafka-2-data:
 354   kafka-3-data:
 355   kafka-4-data: