juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  27       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  28       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  29       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  30         org.apache.kafka.common.security.plain.PlainLoginModule required \
  31         user_schemaregistry="schemaregistry-secret" \
  32         user_connect="connect-secret" \
  33         user_client="client-secret";
  34       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  35       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  36         org.apache.kafka.common.security.plain.PlainLoginModule required \
  37         user_client="client-secret";
  38       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  39     volumes:
  40       - kafka-1-data:/var/lib/kafka/data
  41     ports:
  42       - 9081:9081
  43     stop_grace_period: 120s
  44     depends_on:
  45       - zookeeper
  46
  47   kafka-2:
  48     image: juplo/kafka
  49     environment:
  50       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  51       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
  52       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  53       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
  54       KAFKA_BROKER_ID: 2
  55       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  56       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  57       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  58       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  59       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  60       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  61       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
  62       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
  63       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
  64         org.apache.kafka.common.security.plain.PlainLoginModule required \
  65         user_schemaregistry="schemaregistry-secret" \
  66         user_connect="connect-secret" \
  67         user_client="client-secret";
  68       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
  69       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
  70         org.apache.kafka.common.security.plain.PlainLoginModule required \
  71         user_client="client-secret";
  72       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
  73     volumes:
  74       - kafka-2-data:/var/lib/kafka/data
  75     ports:
  76       - 9092:9082
  77       - 9082:9082
  78     networks:
  79       default:
  80         aliases:
  81           - kafka
  82     stop_grace_period: 120s
  83     depends_on:
  84       - zookeeper
  85
  86   kafka-3:
  87     image: juplo/kafka
  88     environment:
  89       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  90       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
  91       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
  92       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
  93       KAFKA_BROKER_ID: 3
  94       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  95       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  96       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  97       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  98       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  99       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 100       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 101       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 102       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 103         org.apache.kafka.common.security.plain.PlainLoginModule required \
 104         user_schemaregistry="schemaregistry-secret" \
 105         user_connect="connect-secret" \
 106         user_client="client-secret";
 107       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 108       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 109         org.apache.kafka.common.security.plain.PlainLoginModule required \
 110         user_client="client-secret";
 111       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 112     volumes:
 113       - kafka-3-data:/var/lib/kafka/data
 114     ports:
 115       - 9083:9083
 116     stop_grace_period: 120s
 117     depends_on:
 118       - zookeeper
 119
 120   kafka-4:
 121     image: juplo/kafka
 122     environment:
 123       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 124       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
 125       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
 126       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
 127       KAFKA_BROKER_ID: 4
 128       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
 129       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 130       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
 131       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
 132       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 133       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 134       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
 135       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
 136       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
 137         org.apache.kafka.common.security.plain.PlainLoginModule required \
 138         user_schemaregistry="schemaregistry-secret" \
 139         user_connect="connect-secret" \
 140         user_client="client-secret";
 141       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
 142       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
 143         org.apache.kafka.common.security.plain.PlainLoginModule required \
 144         user_client="client-secret";
 145       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
 146     volumes:
 147       - kafka-4-data:/var/lib/kafka/data
 148     ports:
 149       - 9084:9084
 150     stop_grace_period: 120s
 151     depends_on:
 152       - zookeeper
 153
 154   cruise-control:
 155     image: juplo/cruise-control
 156     environment:
 157       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
 158       ZOOKEEPER_CONNECT: zookeeper:2181/
 159     ports:
 160       - "9090:9090"
 161
 162   schema-registry:
 163     image: confluentinc/cp-schema-registry:7.5.1
 164     environment:
 165       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 166       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 167       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 168       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 169       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
 170       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
 171       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
 172         org.apache.kafka.common.security.plain.PlainLoginModule
 173         required
 174         username="schemaregistry"
 175         password="schemaregistry-secret";
 176     ports:
 177       - 8085:8085
 178     depends_on:
 179       - kafka-1
 180       - kafka-2
 181       - kafka-3
 182
 183   connect:
 184     image: confluentinc/cp-kafka-connect:7.5.1
 185     environment:
 186       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 187       CONNECT_REST_PORT: 8083
 188       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 189       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 190       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 191       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 192       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 193       CONNECT_GROUP_ID: kafka-connect
 194       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 195       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 196       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 197       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 198       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 199       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 200       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 201       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 202       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 203       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 204       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 205       CONNECT_PLUGIN_PATH: /usr/share/java/
 206       # Configure the Connect workers to use SASL/PLAIN.
 207       CONNECT_SASL_MECHANISM: PLAIN
 208       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
 209       # JAAS
 210       CONNECT_SASL_JAAS_CONFIG: >
 211         org.apache.kafka.common.security.plain.PlainLoginModule required
 212         username="connect"
 213         password="connect-secret";
 214       # producer
 215       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
 216       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 217       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
 218         org.apache.kafka.common.security.plain.PlainLoginModule required
 219         username="connect"
 220         password="connect-secret";
 221       # consumer
 222       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
 223       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
 224       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
 225         org.apache.kafka.common.security.plain.PlainLoginModule required
 226         username="connect"
 227         password="connect-secret";
 228
 229     ports:
 230       - 8083:8083
 231     depends_on:
 232       - schema-registry
 233
 234   cli:
 235     image: juplo/toolbox
 236     command: sleep infinity
 237     stop_grace_period: 0s
 238     depends_on:
 239       - kafka-1
 240       - kafka-2
 241       - kafka-3
 242
 243   setup:
 244     image: juplo/toolbox
 245     command:
 246       - bash
 247       - -c
 248       - |
 249         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
 250         if [ -e INITIALIZED ]
 251         then
 252           echo -n Bereits konfiguriert:
 253           cat INITIALIZED
 254           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 255           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 256         else
 257           kafka-topics --bootstrap-server kafka:9091 \
 258                        --delete \
 259                        --if-exists \
 260                        --topic test
 261           kafka-topics --bootstrap-server kafka:9091 \
 262                        --create \
 263                        --topic test \
 264                        --partitions 2 \
 265                        --replication-factor 3 \
 266                        --config min.insync.replicas=2 \
 267           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 268           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
 269           kafka-topics --bootstrap-server kafka:9091 \
 270                        --delete \
 271                        --if-exists \
 272                        --topic __CruiseControlMetrics
 273           kafka-topics --bootstrap-server kafka:9091 \
 274                        --create \
 275                        --topic __CruiseControlMetrics \
 276                        --partitions 2 \
 277                        --replication-factor 3 \
 278                        --config min.insync.replicas=2 \
 279           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 280           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
 281         fi \
 282         && date > INITIALIZED
 283     stop_grace_period: 0s
 284     depends_on:
 285       - cli
 286
 287   zoonavigator:
 288     image: elkozmon/zoonavigator:1.1.2
 289     ports:
 290       - "8000:80"
 291     environment:
 292       HTTP_PORT: 80
 293       CONNECTION_JUPLO_NAME: juplo
 294       CONNECTION_JUPLO_CONN: zookeeper:2181
 295       AUTO_CONNECT_CONNECTION_ID: JUPLO
 296     depends_on:
 297       - zookeeper
 298
 299   akhq:
 300     image: tchiotludo/akhq:0.23.0
 301     ports:
 302       - 8888:8080
 303     environment:
 304       AKHQ_CONFIGURATION: |
 305         akhq:
 306           connections:
 307             docker-kafka-server:
 308               properties:
 309                 bootstrap.servers: "kafka:9091"
 310               schema-registry:
 311                 url: "http://schema-registry:8085"
 312               connect:
 313                 - name: "connect"
 314                   url: "http://connect:8083"
 315     depends_on:
 316       - kafka-1
 317       - kafka-2
 318       - kafka-3
 319
 320   producer:
 321     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 322     environment:
 323       producer.throttle-ms: 1
 324       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 325       spring.kafka.client-id: producer
 326       spring.kafka.template.default-topic: test
 327       spring.kafka.producer.properties.linger.ms: 100
 328
 329   consumer-1:
 330     image: juplo/spring-consumer:1.0-SNAPSHOT
 331     environment:
 332       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 333       spring.kafka.client-id: consumer-1
 334
 335   consumer-2:
 336     image: juplo/spring-consumer:1.0-SNAPSHOT
 337     environment:
 338       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 339       spring.kafka.client-id: consumer-2
 340
 341   consumer-3:
 342     image: juplo/spring-consumer:1.0-SNAPSHOT
 343     environment:
 344       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 345       spring.kafka.client-id: consumer-3
 346
 347 volumes:
 348   zookeeper-data:
 349   zookeeper-log:
 350   kafka-1-data:
 351   kafka-2-data:
 352   kafka-3-data:
 353   kafka-4-data: