juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  25       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  26     volumes:
  27       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  28       - kafka-1-data:/var/lib/kafka/data
  29     ports:
  30       - 9081:9081
  31     stop_grace_period: 120s
  32     depends_on:
  33       - zookeeper
  34
  35   kafka-2:
  36     image: confluentinc/cp-kafka:7.5.1
  37     environment:
  38       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  39       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  40       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  41       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  42       KAFKA_BROKER_ID: 2
  43       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  44       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  45       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  46       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  47       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  48     volumes:
  49       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  50       - kafka-2-data:/var/lib/kafka/data
  51     ports:
  52       - 9092:9082
  53       - 9082:9082
  54     networks:
  55       default:
  56         aliases:
  57           - kafka
  58     stop_grace_period: 120s
  59     depends_on:
  60       - zookeeper
  61
  62   kafka-3:
  63     image: confluentinc/cp-kafka:7.5.1
  64     environment:
  65       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  66       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  67       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  68       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  69       KAFKA_BROKER_ID: 3
  70       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  71       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  72       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  73       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  74       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  75     volumes:
  76       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  77       - kafka-3-data:/var/lib/kafka/data
  78     ports:
  79       - 9083:9083
  80     stop_grace_period: 120s
  81     depends_on:
  82       - zookeeper
  83
  84   kafka-4:
  85     image: confluentinc/cp-kafka:7.5.1
  86     environment:
  87       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  88       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9084
  89       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  90       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
  91       KAFKA_BROKER_ID: 4
  92       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  93       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  94       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  95       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  96       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9092
  97     volumes:
  98       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  99       - kafka-4-data:/var/lib/kafka/data
 100     ports:
 101       - 9084:9084
 102     stop_grace_period: 120s
 103     depends_on:
 104       - zookeeper
 105
 106   cruise-control:
 107     image: juplo/cruise-control
 108     environment:
 109       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 110       ZOOKEEPER_CONNECT: zookeeper:2181/
 111     ports:
 112       - "9090:9090"
 113
 114   schema-registry:
 115     image: confluentinc/cp-schema-registry:7.5.1
 116     environment:
 117       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 118       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 119       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 120       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 121     ports:
 122       - 8085:8085
 123     depends_on:
 124       - kafka-1
 125       - kafka-2
 126       - kafka-3
 127
 128   connect:
 129     image: confluentinc/cp-kafka-connect:7.5.1
 130     environment:
 131       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 132       CONNECT_REST_PORT: 8083
 133       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 134       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 135       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 136       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 137       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 138       CONNECT_GROUP_ID: kafka-connect
 139       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 140       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 141       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 142       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 143       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 144       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 145       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 146       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 147       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 148       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 149       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 150       CONNECT_PLUGIN_PATH: /usr/share/java/
 151     ports:
 152       - 8083:8083
 153     depends_on:
 154       - schema-registry
 155
 156   cli:
 157     image: juplo/toolbox
 158     command: sleep infinity
 159     stop_grace_period: 0s
 160     depends_on:
 161       - kafka-1
 162       - kafka-2
 163       - kafka-3
 164
 165   setup:
 166     image: juplo/toolbox
 167     command:
 168       - bash
 169       - -c
 170       - |
 171         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 172         if [ -e INITIALIZED ]
 173         then
 174           echo -n Bereits konfiguriert:
 175           cat INITIALIZED
 176           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 177           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 178         else
 179           kafka-topics --bootstrap-server kafka:9092 \
 180                        --delete \
 181                        --if-exists \
 182                        --topic test
 183           kafka-topics --bootstrap-server kafka:9092 \
 184                        --create \
 185                        --topic test \
 186                        --partitions 2 \
 187                        --replication-factor 3 \
 188                        --config min.insync.replicas=2 \
 189           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 190           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 191           kafka-topics --bootstrap-server kafka:9092 \
 192                        --delete \
 193                        --if-exists \
 194                        --topic __CruiseControlMetrics
 195           kafka-topics --bootstrap-server kafka:9092 \
 196                        --create \
 197                        --topic __CruiseControlMetrics \
 198                        --partitions 2 \
 199                        --replication-factor 3 \
 200                        --config min.insync.replicas=2 \
 201           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
 202           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 203         fi \
 204         && date > INITIALIZED
 205     stop_grace_period: 0s
 206     depends_on:
 207       - cli
 208
 209   zoonavigator:
 210     image: elkozmon/zoonavigator:1.1.2
 211     ports:
 212       - "8000:80"
 213     environment:
 214       HTTP_PORT: 80
 215       CONNECTION_JUPLO_NAME: juplo
 216       CONNECTION_JUPLO_CONN: zookeeper:2181
 217       AUTO_CONNECT_CONNECTION_ID: JUPLO
 218     depends_on:
 219       - zookeeper
 220
 221   akhq:
 222     image: tchiotludo/akhq:0.23.0
 223     ports:
 224       - 8888:8080
 225     environment:
 226       AKHQ_CONFIGURATION: |
 227         akhq:
 228           connections:
 229             docker-kafka-server:
 230               properties:
 231                 bootstrap.servers: "kafka:9092"
 232               schema-registry:
 233                 url: "http://schema-registry:8085"
 234               connect:
 235                 - name: "connect"
 236                   url: "http://connect:8083"
 237     depends_on:
 238       - kafka-1
 239       - kafka-2
 240       - kafka-3
 241
 242   producer:
 243     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
 244     environment:
 245       producer.throttle-ms: 1
 246       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 247       spring.kafka.client-id: producer
 248       spring.kafka.template.default-topic: test
 249       spring.kafka.producer.properties.linger.ms: 100
 250
 251   consumer-1:
 252     image: juplo/spring-consumer:1.0-SNAPSHOT
 253     environment:
 254       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 255       spring.kafka.client-id: consumer-1
 256
 257   consumer-2:
 258     image: juplo/spring-consumer:1.0-SNAPSHOT
 259     environment:
 260       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 261       spring.kafka.client-id: consumer-2
 262
 263   consumer-3:
 264     image: juplo/spring-consumer:1.0-SNAPSHOT
 265     environment:
 266       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
 267       spring.kafka.client-id: consumer-3
 268
 269 volumes:
 270   zookeeper-data:
 271   zookeeper-log:
 272   kafka-1-data:
 273   kafka-2-data:
 274   kafka-3-data:
 275   kafka-4-data: