juplo.de Git - demos/kafka/monitoring/blob - docker-compose.yml

   1 version: '3.2'
   2
   3 services:
   4
   5   zookeeper:
   6     build: zookeeper
   7     environment:
   8       ALLOW_ANONYMOUS_LOGIN: 'yes'
   9       ZOO_ENABLE_PROMETHEUS_METRICS: 'yes'
  10     volumes:
  11       - zookeeper:/bitnami/zookeeper
  12     ports:
  13       - 2181:2181
  14       - 7000:7070
  15
  16   kafka-1:
  17     build: kafka
  18     environment:
  19       KAFKA_ENABLE_KRAFT: 'false'
  20       KAFKA_OPTS:
  21         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  22         -Dcom.sun.management.jmxremote=true
  23         -Dcom.sun.management.jmxremote.port=7000
  24         -Dcom.sun.management.jmxremote.authenticate=false
  25       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  26       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  27       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  28       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  29       KAFKA_BROKER_ID: 1
  30       ALLOW_PLAINTEXT_LISTENER: 'yes'
  31       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  32       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  33       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  34     volumes:
  35       - kafka-1:/bitnami/kafka
  36     ports:
  37       - 9081:9081
  38       - 7001:7000
  39     depends_on:
  40       - zookeeper
  41
  42   kafka-2:
  43     build: kafka
  44     environment:
  45       KAFKA_ENABLE_KRAFT: 'false'
  46       KAFKA_OPTS:
  47         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  48         -Dcom.sun.management.jmxremote=true
  49         -Dcom.sun.management.jmxremote.port=7000
  50         -Dcom.sun.management.jmxremote.authenticate=false
  51       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  52       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  53       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  54       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  55       KAFKA_BROKER_ID: 2
  56       ALLOW_PLAINTEXT_LISTENER: 'yes'
  57       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  58       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  59       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  60     volumes:
  61       - kafka-2:/bitnami/kafka
  62     ports:
  63       - 9092:9082
  64       - 9082:9082
  65       - 7002:7000
  66     networks:
  67       default:
  68         aliases:
  69           - kafka
  70     depends_on:
  71       - zookeeper
  72
  73   kafka-3:
  74     build: kafka
  75     environment:
  76       KAFKA_ENABLE_KRAFT: 'false'
  77       KAFKA_OPTS:
  78         -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
  79         -Dcom.sun.management.jmxremote=true
  80         -Dcom.sun.management.jmxremote.port=7000
  81         -Dcom.sun.management.jmxremote.authenticate=false
  82       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
  83       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  84       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  85       KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  86       KAFKA_BROKER_ID: 3
  87       ALLOW_PLAINTEXT_LISTENER: 'yes'
  88       KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER
  89       KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  90       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false"
  91     volumes:
  92       - kafka-3:/bitnami/kafka
  93     ports:
  94       - 9083:9083
  95       - 7003:7000
  96     depends_on:
  97       - zookeeper
  98
  99   setup:
 100     image: juplo/toolbox
 101     command: >
 102       bash -c "
 103         kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test
 104         kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2
 105         kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 106       "
 107     depends_on:
 108       - kafka-1
 109       - kafka-2
 110       - kafka-3
 111
 112   cli:
 113     image: juplo/toolbox
 114     command: sleep infinity
 115     depends_on:
 116       - kafka-1
 117       - kafka-2
 118       - kafka-3
 119
 120   burrow:
 121     build: burrow
 122     volumes:
 123       - ./burrow.conf:/app/burrow.toml:ro
 124     ports:
 125       - 8080:8080
 126
 127   burrow_exporter:
 128     build: burrow_exporter
 129     environment:
 130       BURROW_ADDR: http://burrow:8080
 131       API_VERSION: 3
 132     ports:
 133       - 8008:8080
 134
 135   kafka-lag-exporter:
 136     image: seglo/kafka-lag-exporter:0.8.2
 137     volumes:
 138       - ./kafka-lag-exporter.conf:/opt/docker/conf/application.conf:ro
 139     ports:
 140       - 8000:8000
 141
 142   kafka-exporter:
 143     image: danielqsj/kafka-exporter:v1.6.0
 144     command: --kafka.server kafka:9092
 145     ports:
 146       - 9308:9308
 147
 148   prometheus:
 149     image: prom/prometheus
 150     ports:
 151       - "9090:9090"
 152     volumes:
 153       - ./prometheus.yml:/etc/prometheus/prometheus.yml
 154
 155   grafana:
 156     image: grafana/grafana:10.0.3
 157     ports:
 158       - "3000:3000"
 159     volumes:
 160       - ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml
 161       - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/automatic.yaml
 162       - ./kafka-overview.json:/var/lib/grafana/dashboards/main-dashboard.json
 163       - grafana:/var/lib/grafana
 164
 165   producer:
 166     image: juplo/simple-producer:1.0-SNAPSHOT
 167     entrypoint: java
 168         -Dcom.sun.management.jmxremote=true
 169         -Dcom.sun.management.jmxremote.port=7000
 170         -Dcom.sun.management.jmxremote.local.only=false
 171         -Dcom.sun.management.jmxremote.authenticate=false
 172         -Dcom.sun.management.jmxremote.ssl=false
 173         -jar /opt/app.jar
 174     command: kafka:9092 test producer
 175     ports:
 176       - 7010:7000
 177
 178   consumer:
 179     image: juplo/simple-consumer:1.0-SNAPSHOT
 180     entrypoint: java
 181         -Dcom.sun.management.jmxremote=true
 182         -Dcom.sun.management.jmxremote.port=7000
 183         -Dcom.sun.management.jmxremote.local.only=false
 184         -Dcom.sun.management.jmxremote.authenticate=false
 185         -Dcom.sun.management.jmxremote.ssl=false
 186         -jar /opt/app.jar
 187     command: kafka:9092 test my-group consumer
 188     ports:
 189       - 7020:7000
 190
 191   cli:
 192     image: juplo/toolbox
 193     command: sleep infinity
 194     depends_on:
 195       - setup
 196
 197 volumes:
 198
 199   zookeeper:
 200   kafka-1:
 201   kafka-2:
 202   kafka-3:
 203   grafana: