Burrow's Zookeeper-Consumer is only needed, if offsets are stored there

[demos/kafka/monitoring] / docker-compose.yml

diff --git a/docker-compose.yml b/docker-compose.yml
index 5622ba8..5ecfff2 100644 (file)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,33 +1,20 @@
 version: '3.2'
+
 services:
-  kafka-0:
-    image: bitnami/kafka:3.4.0
+
+  zookeeper:
+    build: zookeeper
     environment:
-      KAFKA_ENABLE_KRAFT: 'yes'
-      KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw
-      KAFKA_CFG_PROCESS_ROLES: controller
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_CFG_LISTENERS: CONTROLLER://:9092
-      KAFKA_CFG_ADVERTISED_LISTENERS: " "
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
-      KAFKA_BROKER_ID: 0
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092
-      ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
+      ALLOW_ANONYMOUS_LOGIN: 'yes'
 
   kafka-1:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
-      KAFKA_ENABLE_KRAFT: 'yes'
-      KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw
-      KAFKA_CFG_PROCESS_ROLES: broker
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
       KAFKA_BROKER_ID: 1
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
@@ -35,20 +22,16 @@ services:
     ports:
       - 9081:9081
     depends_on:
-      - kafka-0
+      - zookeeper
 
   kafka-2:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
-      KAFKA_ENABLE_KRAFT: 'yes'
-      KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw
-      KAFKA_CFG_PROCESS_ROLES: broker
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
       KAFKA_BROKER_ID: 2
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
@@ -61,20 +44,16 @@ services:
         aliases:
           - kafka
     depends_on:
-      - kafka-0
+      - zookeeper
 
   kafka-3:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
-      KAFKA_ENABLE_KRAFT: 'yes'
-      KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw
-      KAFKA_CFG_PROCESS_ROLES: broker
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
       KAFKA_BROKER_ID: 3
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
@@ -82,5 +61,71 @@ services:
     ports:
       - 9083:9083
     depends_on:
-      - kafka-0
+      - zookeeper
+
+  setup:
+    image: juplo/toolbox
+    command: >
+      bash -c "
+        kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test
+        kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2
+        kafka-topics --bootstrap-server kafka:9092 --describe --topic test
+      "
+    depends_on:
+      - kafka-1
+      - kafka-2
+      - kafka-3
+
+  cli:
+    image: juplo/toolbox
+    command: sleep infinity
+    depends_on:
+      - kafka-1
+      - kafka-2
+      - kafka-3
+
+  burrow:
+    build: burrow
+    volumes:
+      - ./burrow.conf:/app/burrow.toml:ro
+    ports:
+      - 8080:8080
+
+  kafka-lag-exporter:
+    image: seglo/kafka-lag-exporter:0.8.2
+    volumes:
+      - ./kafka-lag-exporter.conf:/opt/docker/conf/application.conf:ro
+    ports:
+      - 8000:8000
+
+  kafka-exporter:
+    image: danielqsj/kafka-exporter:v1.6.0
+    command: --kafka.server kafka:9092
+    ports:
+      - 9308:9308
+
+  prometheus:
+    image: prom/prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+
+  grafana:
+    image: grafana/grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana:/var/lib/grafana
+
+  producer:
+    image: juplo/simple-producer:1.0-SNAPSHOT
+    command: kafka:9092 test producer
+
+  consumer:
+    image: juplo/simple-consumer:1.0-SNAPSHOT
+    command: kafka:9092 test my-group consumer
+
+volumes:
 
+  grafana: