juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: juplo/kafka
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  25       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  26     volumes:
  27       - kafka-1-data:/var/lib/kafka/data
  28     ports:
  29       - 9081:9081
  30     stop_grace_period: 120s
  31     depends_on:
  32       - zookeeper
  33
  34   kafka-2:
  35     image: juplo/kafka
  36     environment:
  37       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  38       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  39       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  40       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  41       KAFKA_BROKER_ID: 2
  42       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  43       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  44       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  45       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  46       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  47     volumes:
  48       - kafka-2-data:/var/lib/kafka/data
  49     ports:
  50       - 9092:9082
  51       - 9082:9082
  52     networks:
  53       default:
  54         aliases:
  55           - kafka
  56     stop_grace_period: 120s
  57     depends_on:
  58       - zookeeper
  59
  60   kafka-3:
  61     image: juplo/kafka
  62     environment:
  63       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  64       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  65       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  66       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  67       KAFKA_BROKER_ID: 3
  68       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  69       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  70       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  71       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  72       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  73     volumes:
  74       - kafka-3-data:/var/lib/kafka/data
  75     ports:
  76       - 9083:9083
  77     stop_grace_period: 120s
  78     depends_on:
  79       - zookeeper
  80
  81   kafka-4:
  82     image: juplo/kafka
  83     environment:
  84       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  85       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9084
  86       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  87       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
  88       KAFKA_BROKER_ID: 4
  89       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  90       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  91       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  92       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  93       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  94     volumes:
  95       - kafka-4-data:/var/lib/kafka/data
  96     ports:
  97       - 9084:9084
  98     stop_grace_period: 120s
  99     depends_on:
 100       - zookeeper
 101
 102   cruise-control:
 103     image: juplo/cruise-control
 104     environment:
 105       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 106       ZOOKEEPER_CONNECT: zookeeper:2181/
 107     ports:
 108       - "9090:9090"
 109
 110   schema-registry:
 111     image: confluentinc/cp-schema-registry:7.5.1
 112     environment:
 113       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 114       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 115       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 116       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 117     ports:
 118       - 8085:8085
 119     depends_on:
 120       - kafka-1
 121       - kafka-2
 122       - kafka-3
 123
 124   connect:
 125     image: confluentinc/cp-kafka-connect:7.5.1
 126     environment:
 127       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 128       CONNECT_REST_PORT: 8083
 129       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 130       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 131       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 132       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 133       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 134       CONNECT_GROUP_ID: kafka-connect
 135       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 136       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 137       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 138       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 139       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 140       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 141       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 142       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 143       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 144       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 145       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 146       CONNECT_PLUGIN_PATH: /usr/share/java/
 147     ports:
 148       - 8083:8083
 149     depends_on:
 150       - schema-registry
 151
 152   cli:
 153     image: juplo/toolbox
 154     command: sleep infinity
 155     stop_grace_period: 0s
 156     depends_on:
 157       - kafka-1
 158       - kafka-2
 159       - kafka-3
 160
 161   setup:
 162     image: juplo/toolbox
 163     command:
 164       - bash
 165       - -c
 166       - |
 167         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 168         if [ -e INITIALIZED ]
 169         then
 170           echo -n Bereits konfiguriert:
 171           cat INITIALIZED
 172           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 173           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 174         else
 175           kafka-topics --bootstrap-server kafka:9092 \
 176                        --delete \
 177                        --if-exists \
 178                        --topic test
 179           kafka-topics --bootstrap-server kafka:9092 \
 180                        --create \
 181                        --topic test \
 182                        --partitions 2 \
 183                        --replication-factor 3 \
 184                        --config min.insync.replicas=2 \
 185           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 186           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 187           kafka-topics --bootstrap-server kafka:9092 \
 188                        --delete \
 189                        --if-exists \
 190                        --topic __CruiseControlMetrics
 191           kafka-topics --bootstrap-server kafka:9092 \
 192                        --create \
 193                        --topic __CruiseControlMetrics \
 194                        --partitions 2 \
 195                        --replication-factor 3 \
 196                        --config min.insync.replicas=2 \
 197           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 198           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 199         fi \
 200         && date > INITIALIZED
 201     stop_grace_period: 0s
 202     depends_on:
 203       - cli
 204
 205   zoonavigator:
 206     image: elkozmon/zoonavigator:1.1.2
 207     ports:
 208       - "8000:80"
 209     environment:
 210       HTTP_PORT: 80
 211       CONNECTION_JUPLO_NAME: juplo
 212       CONNECTION_JUPLO_CONN: zookeeper:2181
 213       AUTO_CONNECT_CONNECTION_ID: JUPLO
 214     depends_on:
 215       - zookeeper
 216
 217   akhq:
 218     image: tchiotludo/akhq:0.23.0
 219     ports:
 220       - 8888:8080
 221     environment:
 222       AKHQ_CONFIGURATION: |
 223         akhq:
 224           connections:
 225             docker-kafka-server:
 226               properties:
 227                 bootstrap.servers: "kafka:9092"
 228               schema-registry:
 229                 url: "http://schema-registry:8085"
 230               connect:
 231                 - name: "connect"
 232                   url: "http://connect:8083"
 233     depends_on:
 234       - kafka-1
 235       - kafka-2
 236       - kafka-3
 237
 238   producer:
 239     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 240     environment:
 241       producer.throttle-ms: 1
 242       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 243       spring.kafka.client-id: producer
 244       spring.kafka.template.default-topic: test
 245       spring.kafka.producer.properties.linger.ms: 100
 246
 247   consumer-1:
 248     image: juplo/spring-consumer:1.0-SNAPSHOT
 249     environment:
 250       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 251       spring.kafka.client-id: consumer-1
 252
 253   consumer-2:
 254     image: juplo/spring-consumer:1.0-SNAPSHOT
 255     environment:
 256       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 257       spring.kafka.client-id: consumer-2
 258
 259   consumer-3:
 260     image: juplo/spring-consumer:1.0-SNAPSHOT
 261     environment:
 262       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 263       spring.kafka.client-id: consumer-3
 264
 265 volumes:
 266   zookeeper-data:
 267   zookeeper-log:
 268   kafka-1-data:
 269   kafka-2-data:
 270   kafka-3-data:
 271   kafka-4-data: