juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24     volumes:
  25       - kafka-1-data:/var/lib/kafka/data
  26     ports:
  27       - 9081:9081
  28     stop_grace_period: 120s
  29     depends_on:
  30       - zookeeper
  31
  32   kafka-2:
  33     image: confluentinc/cp-kafka:7.5.1
  34     environment:
  35       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  36       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  37       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  38       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  39       KAFKA_BROKER_ID: 2
  40       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  41       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  42       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  43     volumes:
  44       - kafka-2-data:/var/lib/kafka/data
  45     ports:
  46       - 9092:9082
  47       - 9082:9082
  48     networks:
  49       default:
  50         aliases:
  51           - kafka
  52     stop_grace_period: 120s
  53     depends_on:
  54       - zookeeper
  55
  56   kafka-3:
  57     image: confluentinc/cp-kafka:7.5.1
  58     environment:
  59       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  60       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  61       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  62       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  63       KAFKA_BROKER_ID: 3
  64       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  65       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  66       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  67     volumes:
  68       - kafka-3-data:/var/lib/kafka/data
  69     ports:
  70       - 9083:9083
  71     stop_grace_period: 120s
  72     depends_on:
  73       - zookeeper
  74
  75   kafka-4:
  76     image: confluentinc/cp-kafka:7.5.1
  77     environment:
  78       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  79       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9084
  80       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  81       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
  82       KAFKA_BROKER_ID: 4
  83       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  84       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  85       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  86     volumes:
  87       - kafka-4-data:/var/lib/kafka/data
  88     ports:
  89       - 9084:9084
  90     stop_grace_period: 120s
  91     depends_on:
  92       - zookeeper
  93
  94   cruise-control:
  95     image: juplo/cruise-control
  96     environment:
  97       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
  98       ZOOKEEPER_CONNECT: zookeeper:2181/
  99     ports:
 100       - "9090:9090"
 101
 102   schema-registry:
 103     image: confluentinc/cp-schema-registry:7.5.1
 104     environment:
 105       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 106       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 107       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 108       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 109     ports:
 110       - 8085:8085
 111     depends_on:
 112       - kafka-1
 113       - kafka-2
 114       - kafka-3
 115
 116   connect:
 117     image: confluentinc/cp-kafka-connect:7.5.1
 118     environment:
 119       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 120       CONNECT_REST_PORT: 8083
 121       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 122       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 123       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 124       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 125       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 126       CONNECT_GROUP_ID: kafka-connect
 127       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 128       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 129       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 130       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 131       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 132       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 133       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 134       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 135       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 136       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 137       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 138       CONNECT_PLUGIN_PATH: /usr/share/java/
 139     ports:
 140       - 8083:8083
 141     depends_on:
 142       - schema-registry
 143
 144   cli:
 145     image: juplo/toolbox
 146     command: sleep infinity
 147     stop_grace_period: 0s
 148     depends_on:
 149       - kafka-1
 150       - kafka-2
 151       - kafka-3
 152
 153   setup:
 154     image: juplo/toolbox
 155     command:
 156       - bash
 157       - -c
 158       - |
 159         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 160         if [ -e INITIALIZED ]
 161         then
 162           echo -n Bereits konfiguriert:
 163           cat INITIALIZED
 164           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 165           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 166         else
 167           kafka-topics --bootstrap-server kafka:9092 \
 168                        --delete \
 169                        --if-exists \
 170                        --topic test
 171           kafka-topics --bootstrap-server kafka:9092 \
 172                        --create \
 173                        --topic test \
 174                        --partitions 2 \
 175                        --replication-factor 3 \
 176                        --config min.insync.replicas=2 \
 177           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 178           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 179           kafka-topics --bootstrap-server kafka:9092 \
 180                        --delete \
 181                        --if-exists \
 182                        --topic __CruiseControlMetrics
 183           kafka-topics --bootstrap-server kafka:9092 \
 184                        --create \
 185                        --topic __CruiseControlMetrics \
 186                        --partitions 2 \
 187                        --replication-factor 3 \
 188                        --config min.insync.replicas=2 \
 189           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 190           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 191         fi \
 192         && date > INITIALIZED
 193     stop_grace_period: 0s
 194     depends_on:
 195       - cli
 196
 197   zoonavigator:
 198     image: elkozmon/zoonavigator:1.1.2
 199     ports:
 200       - "8000:80"
 201     environment:
 202       HTTP_PORT: 80
 203       CONNECTION_JUPLO_NAME: juplo
 204       CONNECTION_JUPLO_CONN: zookeeper:2181
 205       AUTO_CONNECT_CONNECTION_ID: JUPLO
 206     depends_on:
 207       - zookeeper
 208
 209   akhq:
 210     image: tchiotludo/akhq:0.23.0
 211     ports:
 212       - 8888:8080
 213     environment:
 214       AKHQ_CONFIGURATION: |
 215         akhq:
 216           connections:
 217             docker-kafka-server:
 218               properties:
 219                 bootstrap.servers: "kafka:9092"
 220               schema-registry:
 221                 url: "http://schema-registry:8085"
 222               connect:
 223                 - name: "connect"
 224                   url: "http://connect:8083"
 225     depends_on:
 226       - kafka-1
 227       - kafka-2
 228       - kafka-3
 229
 230   producer:
 231     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 232     environment:
 233       producer.throttle-ms: 1
 234       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 235       spring.kafka.client-id: producer
 236       spring.kafka.template.default-topic: test
 237       spring.kafka.producer.properties.linger.ms: 100
 238
 239   consumer-1:
 240     image: juplo/spring-consumer:1.0-SNAPSHOT
 241     environment:
 242       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 243       spring.kafka.client-id: consumer-1
 244
 245   consumer-2:
 246     image: juplo/spring-consumer:1.0-SNAPSHOT
 247     environment:
 248       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 249       spring.kafka.client-id: consumer-2
 250
 251   consumer-3:
 252     image: juplo/spring-consumer:1.0-SNAPSHOT
 253     environment:
 254       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 255       spring.kafka.client-id: consumer-3
 256
 257 volumes:
 258   zookeeper-data:
 259   zookeeper-log:
 260   kafka-1-data:
 261   kafka-2-data:
 262   kafka-3-data:
 263   kafka-4-data: