juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  25     volumes:
  26       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  27       - kafka-1-data:/var/lib/kafka/data
  28     ports:
  29       - 9081:9081
  30     stop_grace_period: 120s
  31     depends_on:
  32       - zookeeper
  33
  34   kafka-2:
  35     image: confluentinc/cp-kafka:7.5.1
  36     environment:
  37       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  38       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  39       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  40       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  41       KAFKA_BROKER_ID: 2
  42       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  43       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  44       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  45       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  46     volumes:
  47       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  48       - kafka-2-data:/var/lib/kafka/data
  49     ports:
  50       - 9092:9082
  51       - 9082:9082
  52     networks:
  53       default:
  54         aliases:
  55           - kafka
  56     stop_grace_period: 120s
  57     depends_on:
  58       - zookeeper
  59
  60   kafka-3:
  61     image: confluentinc/cp-kafka:7.5.1
  62     environment:
  63       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  64       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  65       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  66       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  67       KAFKA_BROKER_ID: 3
  68       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  69       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  70       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  71       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  72     volumes:
  73       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  74       - kafka-3-data:/var/lib/kafka/data
  75     ports:
  76       - 9083:9083
  77     stop_grace_period: 120s
  78     depends_on:
  79       - zookeeper
  80
  81   cruise-control:
  82     image: juplo/cruise-control
  83     environment:
  84       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
  85       ZOOKEEPER_CONNECT: zookeeper:2181/
  86     ports:
  87       - "9090:9090"
  88
  89   schema-registry:
  90     image: confluentinc/cp-schema-registry:7.5.1
  91     environment:
  92       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
  93       SCHEMA_REGISTRY_HOST_NAME: schema-registry
  94       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
  95       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
  96     ports:
  97       - 8085:8085
  98     depends_on:
  99       - kafka-1
 100       - kafka-2
 101       - kafka-3
 102
 103   connect:
 104     image: confluentinc/cp-kafka-connect:7.5.1
 105     environment:
 106       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 107       CONNECT_REST_PORT: 8083
 108       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 109       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 110       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 111       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 112       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 113       CONNECT_GROUP_ID: kafka-connect
 114       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 115       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 116       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 117       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 118       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 119       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 120       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 121       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 122       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 123       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 124       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 125       CONNECT_PLUGIN_PATH: /usr/share/java/
 126     ports:
 127       - 8083:8083
 128     depends_on:
 129       - schema-registry
 130
 131   cli:
 132     image: juplo/toolbox
 133     command: sleep infinity
 134     stop_grace_period: 0s
 135     depends_on:
 136       - kafka-1
 137       - kafka-2
 138       - kafka-3
 139
 140   setup:
 141     image: juplo/toolbox
 142     command:
 143       - bash
 144       - -c
 145       - |
 146         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 147         if [ -e INITIALIZED ]
 148         then
 149           echo -n Bereits konfiguriert:
 150           cat INITIALIZED
 151           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 152           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 153         else
 154           kafka-topics --bootstrap-server kafka:9092 \
 155                        --delete \
 156                        --if-exists \
 157                        --topic test
 158           kafka-topics --bootstrap-server kafka:9092 \
 159                        --create \
 160                        --topic test \
 161                        --partitions 2 \
 162                        --replication-factor 3 \
 163                        --config min.insync.replicas=2 \
 164           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 165           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 166           kafka-topics --bootstrap-server kafka:9092 \
 167                        --delete \
 168                        --if-exists \
 169                        --topic __CruiseControlMetrics
 170           kafka-topics --bootstrap-server kafka:9092 \
 171                        --create \
 172                        --topic __CruiseControlMetrics \
 173                        --partitions 2 \
 174                        --replication-factor 3 \
 175                        --config min.insync.replicas=2 \
 176           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 177           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 178         fi \
 179         && date > INITIALIZED
 180     stop_grace_period: 0s
 181     depends_on:
 182       - cli
 183
 184   zoonavigator:
 185     image: elkozmon/zoonavigator:1.1.2
 186     ports:
 187       - "8000:80"
 188     environment:
 189       HTTP_PORT: 80
 190       CONNECTION_JUPLO_NAME: juplo
 191       CONNECTION_JUPLO_CONN: zookeeper:2181
 192       AUTO_CONNECT_CONNECTION_ID: JUPLO
 193     depends_on:
 194       - zookeeper
 195
 196   akhq:
 197     image: tchiotludo/akhq:0.23.0
 198     ports:
 199       - 8888:8080
 200     environment:
 201       AKHQ_CONFIGURATION: |
 202         akhq:
 203           connections:
 204             docker-kafka-server:
 205               properties:
 206                 bootstrap.servers: "kafka:9092"
 207               schema-registry:
 208                 url: "http://schema-registry:8085"
 209               connect:
 210                 - name: "connect"
 211                   url: "http://connect:8083"
 212     depends_on:
 213       - kafka-1
 214       - kafka-2
 215       - kafka-3
 216
 217   cruise-control-ui:
 218     image: juplo/cruise-control-ui
 219     volumes:
 220       - ./config.csv:/usr/share/nginx/html/static/config.csv
 221     ports:
 222       - "9080:80"
 223
 224   producer:
 225     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 226     environment:
 227       producer.throttle-ms: 1
 228       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 229       spring.kafka.client-id: producer
 230       spring.kafka.template.default-topic: test
 231       spring.kafka.producer.properties.linger.ms: 100
 232
 233   consumer-1:
 234     image: juplo/spring-consumer:1.0-SNAPSHOT
 235     environment:
 236       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 237       spring.kafka.client-id: consumer-1
 238
 239   consumer-2:
 240     image: juplo/spring-consumer:1.0-SNAPSHOT
 241     environment:
 242       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 243       spring.kafka.client-id: consumer-2
 244
 245   consumer-3:
 246     image: juplo/spring-consumer:1.0-SNAPSHOT
 247     environment:
 248       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 249       spring.kafka.client-id: consumer-3
 250
 251 volumes:
 252   zookeeper-data:
 253   zookeeper-log:
 254   kafka-1-data:
 255   kafka-2-data:
 256   kafka-3-data: