juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  27       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  28       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SECURITY_PROTOCOL: SASL_PLAINTEXT
  29       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_MECHANISM: PLAIN
  30       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="ccreporter" password="ccreporter-secret";
  31       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  32       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  33         org.apache.kafka.common.security.plain.PlainLoginModule required \
  34         user_schemaregistry="schemaregistry-secret" \
  35         user_connect="connect-secret" \
  36         user_ccreporter="ccreporter-secret" \
  37         user_ui="ui-secret" \
  38         user_client="client-secret";
  39       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  40       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  41         org.apache.kafka.common.security.plain.PlainLoginModule required \
  42         user_client="client-secret";
  43       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  44     volumes:
  45       - kafka-1-data:/var/lib/kafka/data
  46     ports:
  47       - 9081:9081
  48     stop_grace_period: 120s
  49     depends_on:
  50       - zookeeper
  51
  52   kafka-2:
  53     image: juplo/kafka
  54     environment:
  55       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  56       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  57       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  58       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  59       KAFKA_BROKER_ID: 2
  60       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  61       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  62       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  63       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  64       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  65       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  66       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  67       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SECURITY_PROTOCOL: SASL_PLAINTEXT
  68       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_MECHANISM: PLAIN
  69       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="ccreporter" password="ccreporter-secret";
  70       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  71       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  72         org.apache.kafka.common.security.plain.PlainLoginModule required \
  73         user_schemaregistry="schemaregistry-secret" \
  74         user_connect="connect-secret" \
  75         user_ccreporter="ccreporter-secret" \
  76         user_ui="ui-secret" \
  77         user_client="client-secret";
  78       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  79       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  80         org.apache.kafka.common.security.plain.PlainLoginModule required \
  81         user_client="client-secret";
  82       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  83     volumes:
  84       - kafka-2-data:/var/lib/kafka/data
  85     ports:
  86       - 9092:9082
  87       - 9082:9082
  88     networks:
  89       default:
  90         aliases:
  91           - kafka
  92     stop_grace_period: 120s
  93     depends_on:
  94       - zookeeper
  95
  96   kafka-3:
  97     image: juplo/kafka
  98     environment:
  99       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 100       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
 101       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 102       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
 103       KAFKA_BROKER_ID: 3
 104       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 105       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 106       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 107       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 108       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 109       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 110       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
 111       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 112       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_MECHANISM: PLAIN
 113       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="ccreporter" password="ccreporter-secret";
 114       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 115       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 116         org.apache.kafka.common.security.plain.PlainLoginModule required \
 117         user_schemaregistry="schemaregistry-secret" \
 118         user_connect="connect-secret" \
 119         user_ccreporter="ccreporter-secret" \
 120         user_ui="ui-secret" \
 121         user_client="client-secret";
 122       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 123       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 124         org.apache.kafka.common.security.plain.PlainLoginModule required \
 125         user_client="client-secret";
 126       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 127     volumes:
 128       - kafka-3-data:/var/lib/kafka/data
 129     ports:
 130       - 9083:9083
 131     stop_grace_period: 120s
 132     depends_on:
 133       - zookeeper
 134
 135   kafka-4:
 136     image: juplo/kafka
 137     environment:
 138       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 139       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 140       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 141       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 142       KAFKA_BROKER_ID: 4
 143       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 144       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 145       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 146       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 147       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 148       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 149       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
 150       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 151       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_MECHANISM: PLAIN
 152       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="ccreporter" password="ccreporter-secret";
 153       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 154       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 155         org.apache.kafka.common.security.plain.PlainLoginModule required \
 156         user_schemaregistry="schemaregistry-secret" \
 157         user_connect="connect-secret" \
 158         user_ccreporter="ccreporter-secret" \
 159         user_ui="ui-secret" \
 160         user_client="client-secret";
 161       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 162       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 163         org.apache.kafka.common.security.plain.PlainLoginModule required \
 164         user_client="client-secret";
 165       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 166     volumes:
 167       - kafka-4-data:/var/lib/kafka/data
 168     ports:
 169       - 9084:9084
 170     stop_grace_period: 120s
 171     depends_on:
 172       - zookeeper
 173
 174   cruise-control:
 175     image: juplo/cruise-control
 176     environment:
 177       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 178       ZOOKEEPER_CONNECT: zookeeper:2181/
 179     ports:
 180       - "9090:9090"
 181
 182   schema-registry:
 183     image: confluentinc/cp-schema-registry:7.5.1
 184     environment:
 185       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 186       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 187       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 188       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 189       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 190       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 191       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 192         org.apache.kafka.common.security.plain.PlainLoginModule
 193         required
 194         username="schemaregistry"
 195         password="schemaregistry-secret";
 196     ports:
 197       - 8085:8085
 198     depends_on:
 199       - kafka-1
 200       - kafka-2
 201       - kafka-3
 202
 203   connect:
 204     image: confluentinc/cp-kafka-connect:7.5.1
 205     environment:
 206       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 207       CONNECT_REST_PORT: 8083
 208       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 209       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 210       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 211       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 212       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 213       CONNECT_GROUP_ID: kafka-connect
 214       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 215       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 216       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 217       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 218       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 219       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 220       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 221       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 222       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 223       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 224       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 225       CONNECT_PLUGIN_PATH: /usr/share/java/
 226       # Configure the Connect workers to use SASL/PLAIN.
 227       CONNECT_SASL_MECHANISM: PLAIN
 228       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 229       # JAAS
 230       CONNECT_SASL_JAAS_CONFIG: >
 231         org.apache.kafka.common.security.plain.PlainLoginModule required
 232         username="connect"
 233         password="connect-secret";
 234       # producer
 235       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 236       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 237       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 238         org.apache.kafka.common.security.plain.PlainLoginModule required
 239         username="connect"
 240         password="connect-secret";
 241       # consumer
 242       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 243       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 244       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 245         org.apache.kafka.common.security.plain.PlainLoginModule required
 246         username="connect"
 247         password="connect-secret";
 248
 249     ports:
 250       - 8083:8083
 251     depends_on:
 252       - schema-registry
 253
 254   cli:
 255     image: juplo/toolbox
 256     command: sleep infinity
 257     stop_grace_period: 0s
 258     depends_on:
 259       - kafka-1
 260       - kafka-2
 261       - kafka-3
 262
 263   setup:
 264     image: juplo/toolbox
 265     command:
 266       - bash
 267       - -c
 268       - |
 269         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 270         if [ -e INITIALIZED ]
 271         then
 272           echo -n Bereits konfiguriert:
 273           cat INITIALIZED
 274           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 275           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 276         else
 277           kafka-topics --bootstrap-server kafka:9091 \
 278                        --delete \
 279                        --if-exists \
 280                        --topic test
 281           kafka-topics --bootstrap-server kafka:9091 \
 282                        --create \
 283                        --topic test \
 284                        --partitions 2 \
 285                        --replication-factor 3 \
 286                        --config min.insync.replicas=2 \
 287           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 288           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 289           kafka-topics --bootstrap-server kafka:9091 \
 290                        --delete \
 291                        --if-exists \
 292                        --topic __CruiseControlMetrics
 293           kafka-topics --bootstrap-server kafka:9091 \
 294                        --create \
 295                        --topic __CruiseControlMetrics \
 296                        --partitions 2 \
 297                        --replication-factor 3 \
 298                        --config min.insync.replicas=2 \
 299           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 300           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 301         fi \
 302         && date > INITIALIZED
 303     stop_grace_period: 0s
 304     depends_on:
 305       - cli
 306
 307   zoonavigator:
 308     image: elkozmon/zoonavigator:1.1.2
 309     ports:
 310       - "8000:80"
 311     environment:
 312       HTTP_PORT: 80
 313       CONNECTION_JUPLO_NAME: juplo
 314       CONNECTION_JUPLO_CONN: zookeeper:2181
 315       AUTO_CONNECT_CONNECTION_ID: JUPLO
 316     depends_on:
 317       - zookeeper
 318
 319   akhq:
 320     image: tchiotludo/akhq:0.23.0
 321     ports:
 322       - 8888:8080
 323     environment:
 324       AKHQ_CONFIGURATION: |
 325         akhq:
 326           connections:
 327             docker-kafka-server:
 328               properties:
 329                 bootstrap.servers: "kafka:9092"
 330                 security.protocol: SASL_PLAINTEXT
 331                 sasl.mechanism: PLAIN
 332                 sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="ui" password="ui-secret";
 333               schema-registry:
 334                 url: "http://schema-registry:8085"
 335               connect:
 336                 - name: "connect"
 337                   url: "http://connect:8083"
 338     depends_on:
 339       - kafka-1
 340       - kafka-2
 341       - kafka-3
 342
 343   producer:
 344     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 345     environment:
 346       producer.throttle-ms: 1
 347       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 348       spring.kafka.client-id: producer
 349       spring.kafka.template.default-topic: test
 350       spring.kafka.producer.properties.linger.ms: 100
 351
 352   consumer-1:
 353     image: juplo/spring-consumer:1.0-SNAPSHOT
 354     environment:
 355       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 356       spring.kafka.client-id: consumer-1
 357
 358   consumer-2:
 359     image: juplo/spring-consumer:1.0-SNAPSHOT
 360     environment:
 361       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 362       spring.kafka.client-id: consumer-2
 363
 364   consumer-3:
 365     image: juplo/spring-consumer:1.0-SNAPSHOT
 366     environment:
 367       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 368       spring.kafka.client-id: consumer-3
 369
 370 volumes:
 371   zookeeper-data:
 372   zookeeper-log:
 373   kafka-1-data:
 374   kafka-2-data:
 375   kafka-3-data:
 376   kafka-4-data: