juplo.de Git - demos/kafka/monitoring/blob - docker-compose.yml

   1 version: '3.2'
   2
   3 services:
   4
   5   zookeeper:
   6     build: zookeeper
   7     environment:
   8       ALLOW_ANONYMOUS_LOGIN: 'yes'
   9       ZOO_ENABLE_PROMETHEUS_METRICS: 'yes'
  10     volumes:
  11       - zookeeper:/bitnami/zookeeper
  12     ports:
  13       - 2181:2181
  14       - 7000:7070
  15
  16   kafka-1:
  17     build: kafka
  18     environment:
  19       KAFKA_ENABLE_KRAFT: 'false'
  20       KAFKA_OPTS:
  21         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  22         -Dcom.sun.management.jmxremote=true
  23         -Dcom.sun.management.jmxremote.port=7000
  24         -Dcom.sun.management.jmxremote.authenticate=false
  25       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  26       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  27       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  28       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  29       KAFKA_BROKER_ID: 1
  30       ALLOW_PLAINTEXT_LISTENER: 'yes'
  31       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  32       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  33       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  34     volumes:
  35       - kafka-1:/bitnami/kafka
  36     ports:
  37       - 9081:9081
  38       - 7001:7000
  39     depends_on:
  40       - zookeeper
  41
  42   kafka-2:
  43     build: kafka
  44     environment:
  45       KAFKA_ENABLE_KRAFT: 'false'
  46       KAFKA_OPTS:
  47         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  48         -Dcom.sun.management.jmxremote=true
  49         -Dcom.sun.management.jmxremote.port=7000
  50         -Dcom.sun.management.jmxremote.authenticate=false
  51       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  52       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  53       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  54       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  55       KAFKA_BROKER_ID: 2
  56       ALLOW_PLAINTEXT_LISTENER: 'yes'
  57       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  58       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  59       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  60     volumes:
  61       - kafka-2:/bitnami/kafka
  62     ports:
  63       - 9092:9082
  64       - 9082:9082
  65       - 7002:7000
  66     networks:
  67       default:
  68         aliases:
  69           - kafka
  70     depends_on:
  71       - zookeeper
  72
  73   kafka-3:
  74     build: kafka
  75     environment:
  76       KAFKA_ENABLE_KRAFT: 'false'
  77       KAFKA_OPTS:
  78         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  79         -Dcom.sun.management.jmxremote=true
  80         -Dcom.sun.management.jmxremote.port=7000
  81         -Dcom.sun.management.jmxremote.authenticate=false
  82       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  83       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  84       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  85       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  86       KAFKA_BROKER_ID: 3
  87       ALLOW_PLAINTEXT_LISTENER: 'yes'
  88       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  89       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  90       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  91     volumes:
  92       - kafka-3:/bitnami/kafka
  93     ports:
  94       - 9083:9083
  95       - 7003:7000
  96     depends_on:
  97       - zookeeper
  98
  99   kafka-4:
 100     build: kafka
 101     environment:
 102       KAFKA_ENABLE_KRAFT: 'false'
 103       KAFKA_OPTS:
 104         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
 105         -Dcom.sun.management.jmxremote=true
 106         -Dcom.sun.management.jmxremote.port=7000
 107         -Dcom.sun.management.jmxremote.authenticate=false
 108       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
 109       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9084
 110       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
 111       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084
 112       KAFKA_BROKER_ID: 4
 113       ALLOW_PLAINTEXT_LISTENER: 'yes'
 114       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
 115       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 116       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
 117     volumes:
 118       - kafka-4:/bitnami/kafka
 119     ports:
 120       - 9084:9084
 121       - 7004:7000
 122     depends_on:
 123       - zookeeper
 124
 125   kafka-5:
 126     build: kafka
 127     environment:
 128       KAFKA_ENABLE_KRAFT: 'false'
 129       KAFKA_OPTS:
 130         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
 131         -Dcom.sun.management.jmxremote=true
 132         -Dcom.sun.management.jmxremote.port=7000
 133         -Dcom.sun.management.jmxremote.authenticate=false
 134       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
 135       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9085
 136       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
 137       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-5:9092, LOCALHOST://localhost:9085
 138       KAFKA_BROKER_ID: 5
 139       ALLOW_PLAINTEXT_LISTENER: 'yes'
 140       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
 141       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
 142       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
 143     volumes:
 144       - kafka-5:/bitnami/kafka
 145     ports:
 146       - 9085:9085
 147       - 7005:7000
 148     depends_on:
 149       - zookeeper
 150
 151   setup:
 152     image: juplo/toolbox
 153     command: >
 154       bash -c "
 155         kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test
 156         kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 33 --replication-factor 3 --config min.insync.replicas=2
 157         kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 158       "
 159     depends_on:
 160       - kafka-1
 161       - kafka-2
 162       - kafka-3
 163       - kafka-4
 164       - kafka-5
 165
 166   cli:
 167     image: juplo/toolbox
 168     command: sleep infinity
 169     depends_on:
 170       - kafka-1
 171       - kafka-2
 172       - kafka-3
 173       - kafka-4
 174       - kafka-5
 175
 176   burrow:
 177     build: burrow
 178     volumes:
 179       - ./burrow.conf:/app/burrow.toml:ro
 180     ports:
 181       - 8080:8080
 182
 183   burrow_exporter:
 184     build: burrow_exporter
 185     environment:
 186       BURROW_ADDR: http://burrow:8080
 187       API_VERSION: 3
 188     ports:
 189       - 8008:8080
 190
 191   kafka-lag-exporter:
 192     image: seglo/kafka-lag-exporter:0.8.2
 193     volumes:
 194       - ./kafka-lag-exporter.conf:/opt/docker/conf/application.conf:ro
 195     ports:
 196       - 8000:8000
 197
 198   kafka-exporter:
 199     image: danielqsj/kafka-exporter:v1.6.0
 200     command: --kafka.server kafka:9092
 201     ports:
 202       - 9308:9308
 203
 204   prometheus:
 205     image: prom/prometheus
 206     ports:
 207       - "9090:9090"
 208     volumes:
 209       - ./prometheus.yml:/etc/prometheus/prometheus.yml
 210
 211   grafana:
 212     image: grafana/grafana:10.0.3
 213     ports:
 214       - "3000:3000"
 215     environment:
 216       GF_SECURITY_ADMIN_USER: juplo
 217       GF_SECURITY_ADMIN_PASSWORD: juplo
 218     volumes:
 219       - ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml
 220       - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/automatic.yaml
 221       - ./kafka-brokers.json:/var/lib/grafana/dashboards/main-dashboard.json
 222       - grafana:/var/lib/grafana
 223
 224   producer:
 225     image: juplo/simple-producer:1.0-SNAPSHOT
 226     entrypoint: >-
 227         java
 228         -Dcom.sun.management.jmxremote=true
 229         -Dcom.sun.management.jmxremote.port=7000
 230         -Dcom.sun.management.jmxremote.local.only=false
 231         -Dcom.sun.management.jmxremote.authenticate=false
 232         -Dcom.sun.management.jmxremote.ssl=false
 233         -jar /opt/app.jar
 234     command: kafka:9092 test producer
 235     ports:
 236       - 7010:7000
 237
 238   consumer:
 239     image: juplo/simple-consumer:1.0-SNAPSHOT
 240     entrypoint: >-
 241         java
 242         -Dcom.sun.management.jmxremote=true
 243         -Dcom.sun.management.jmxremote.port=7000
 244         -Dcom.sun.management.jmxremote.local.only=false
 245         -Dcom.sun.management.jmxremote.authenticate=false
 246         -Dcom.sun.management.jmxremote.ssl=false
 247         -jar /opt/app.jar
 248     command: kafka:9092 test my-group consumer
 249     ports:
 250       - 7020:7000
 251
 252 volumes:
 253
 254   zookeeper:
 255   kafka-1:
 256   kafka-2:
 257   kafka-3:
 258   kafka-4:
 259   kafka-5:
 260   grafana: