juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  27       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  28     volumes:
  29       - kafka-1-data:/var/lib/kafka/data
  30     ports:
  31       - 9081:9081
  32     stop_grace_period: 120s
  33     depends_on:
  34       - zookeeper
  35
  36   kafka-2:
  37     image: juplo/kafka
  38     environment:
  39       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  40       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  41       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  42       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  43       KAFKA_BROKER_ID: 2
  44       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  45       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  46       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  47       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  48       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  49       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  50       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  51     volumes:
  52       - kafka-2-data:/var/lib/kafka/data
  53     ports:
  54       - 9092:9082
  55       - 9082:9082
  56     networks:
  57       default:
  58         aliases:
  59           - kafka
  60     stop_grace_period: 120s
  61     depends_on:
  62       - zookeeper
  63
  64   kafka-3:
  65     image: juplo/kafka
  66     environment:
  67       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  68       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  69       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  70       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  71       KAFKA_BROKER_ID: 3
  72       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  73       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  74       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  75       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  76       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  77       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  78       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  79     volumes:
  80       - kafka-3-data:/var/lib/kafka/data
  81     ports:
  82       - 9083:9083
  83     stop_grace_period: 120s
  84     depends_on:
  85       - zookeeper
  86
  87   kafka-4:
  88     image: juplo/kafka
  89     environment:
  90       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  91       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9084
  92       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  93       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
  94       KAFKA_BROKER_ID: 4
  95       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  96       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  97       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  98       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  99       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
 100       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
 101       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
 102     volumes:
 103       - kafka-4-data:/var/lib/kafka/data
 104     ports:
 105       - 9084:9084
 106     stop_grace_period: 120s
 107     depends_on:
 108       - zookeeper
 109
 110   cruise-control:
 111     image: juplo/cruise-control
 112     environment:
 113       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 114       ZOOKEEPER_CONNECT: zookeeper:2181/
 115     ports:
 116       - "9090:9090"
 117
 118   schema-registry:
 119     image: confluentinc/cp-schema-registry:7.5.1
 120     environment:
 121       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 122       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 123       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 124       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 125     ports:
 126       - 8085:8085
 127     depends_on:
 128       - kafka-1
 129       - kafka-2
 130       - kafka-3
 131
 132   connect:
 133     image: confluentinc/cp-kafka-connect:7.5.1
 134     environment:
 135       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 136       CONNECT_REST_PORT: 8083
 137       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 138       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 139       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 140       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 141       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 142       CONNECT_GROUP_ID: kafka-connect
 143       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 144       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 145       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 146       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 147       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 148       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 149       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 150       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 151       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 152       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 153       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 154       CONNECT_PLUGIN_PATH: /usr/share/java/
 155     ports:
 156       - 8083:8083
 157     depends_on:
 158       - schema-registry
 159
 160   cli:
 161     image: juplo/toolbox
 162     command: sleep infinity
 163     stop_grace_period: 0s
 164     depends_on:
 165       - kafka-1
 166       - kafka-2
 167       - kafka-3
 168
 169   setup:
 170     image: juplo/toolbox
 171     command:
 172       - bash
 173       - -c
 174       - |
 175         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 176         if [ -e INITIALIZED ]
 177         then
 178           echo -n Bereits konfiguriert:
 179           cat INITIALIZED
 180           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 181           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 182         else
 183           kafka-topics --bootstrap-server kafka:9092 \
 184                        --delete \
 185                        --if-exists \
 186                        --topic test
 187           kafka-topics --bootstrap-server kafka:9092 \
 188                        --create \
 189                        --topic test \
 190                        --partitions 2 \
 191                        --replication-factor 3 \
 192                        --config min.insync.replicas=2 \
 193           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 194           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 195           kafka-topics --bootstrap-server kafka:9092 \
 196                        --delete \
 197                        --if-exists \
 198                        --topic __CruiseControlMetrics
 199           kafka-topics --bootstrap-server kafka:9092 \
 200                        --create \
 201                        --topic __CruiseControlMetrics \
 202                        --partitions 2 \
 203                        --replication-factor 3 \
 204                        --config min.insync.replicas=2 \
 205           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 206           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 207         fi \
 208         && date > INITIALIZED
 209     stop_grace_period: 0s
 210     depends_on:
 211       - cli
 212
 213   zoonavigator:
 214     image: elkozmon/zoonavigator:1.1.2
 215     ports:
 216       - "8000:80"
 217     environment:
 218       HTTP_PORT: 80
 219       CONNECTION_JUPLO_NAME: juplo
 220       CONNECTION_JUPLO_CONN: zookeeper:2181
 221       AUTO_CONNECT_CONNECTION_ID: JUPLO
 222     depends_on:
 223       - zookeeper
 224
 225   akhq:
 226     image: tchiotludo/akhq:0.23.0
 227     ports:
 228       - 8888:8080
 229     environment:
 230       AKHQ_CONFIGURATION: |
 231         akhq:
 232           connections:
 233             docker-kafka-server:
 234               properties:
 235                 bootstrap.servers: "kafka:9092"
 236               schema-registry:
 237                 url: "http://schema-registry:8085"
 238               connect:
 239                 - name: "connect"
 240                   url: "http://connect:8083"
 241     depends_on:
 242       - kafka-1
 243       - kafka-2
 244       - kafka-3
 245
 246   producer:
 247     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 248     environment:
 249       producer.throttle-ms: 1
 250       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 251       spring.kafka.client-id: producer
 252       spring.kafka.template.default-topic: test
 253       spring.kafka.producer.properties.linger.ms: 100
 254
 255   consumer-1:
 256     image: juplo/spring-consumer:1.0-SNAPSHOT
 257     environment:
 258       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 259       spring.kafka.client-id: consumer-1
 260
 261   consumer-2:
 262     image: juplo/spring-consumer:1.0-SNAPSHOT
 263     environment:
 264       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 265       spring.kafka.client-id: consumer-2
 266
 267   consumer-3:
 268     image: juplo/spring-consumer:1.0-SNAPSHOT
 269     environment:
 270       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 271       spring.kafka.client-id: consumer-3
 272
 273 volumes:
 274   zookeeper-data:
 275   zookeeper-log:
 276   kafka-1-data:
 277   kafka-2-data:
 278   kafka-3-data:
 279   kafka-4-data: