From: Kai Moritz Date: Tue, 26 Sep 2023 20:42:32 +0000 (+0200) Subject: Copied `docker-compose.yml` to subfolder `kubernetes` X-Git-Url: https://juplo.de/gitweb/?a=commitdiff_plain;h=244269990950495c83b9173df64fe4b056105ffd;p=demos%2Fkafka%2Fmonitoring Copied `docker-compose.yml` to subfolder `kubernetes` --- diff --git a/kubernetes/docker-compose.yml b/kubernetes/docker-compose.yml new file mode 100644 index 0000000..e32cbde --- /dev/null +++ b/kubernetes/docker-compose.yml @@ -0,0 +1,260 @@ +version: '3.2' + +services: + + zookeeper: + build: zookeeper + environment: + ALLOW_ANONYMOUS_LOGIN: 'yes' + ZOO_ENABLE_PROMETHEUS_METRICS: 'yes' + volumes: + - zookeeper:/bitnami/zookeeper + ports: + - 2181:2181 + - 7000:7070 + + kafka-1: + build: kafka + environment: + KAFKA_ENABLE_KRAFT: 'false' + KAFKA_OPTS: + -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.authenticate=false + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 + KAFKA_BROKER_ID: 1 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + volumes: + - kafka-1:/bitnami/kafka + ports: + - 9081:9081 + - 7001:7000 + depends_on: + - zookeeper + + kafka-2: + build: kafka + environment: + KAFKA_ENABLE_KRAFT: 'false' + KAFKA_OPTS: + -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.authenticate=false + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 + KAFKA_BROKER_ID: 2 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + volumes: + - kafka-2:/bitnami/kafka + ports: + - 9092:9082 + - 9082:9082 + - 7002:7000 + networks: + default: + aliases: + - kafka + depends_on: + - zookeeper + + kafka-3: + build: kafka + environment: + KAFKA_ENABLE_KRAFT: 'false' + KAFKA_OPTS: + -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.authenticate=false + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 + KAFKA_BROKER_ID: 3 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + volumes: + - kafka-3:/bitnami/kafka + ports: + - 9083:9083 + - 7003:7000 + depends_on: + - zookeeper + + kafka-4: + build: kafka + environment: + KAFKA_ENABLE_KRAFT: 'false' + KAFKA_OPTS: + -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.authenticate=false + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9084 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-4:9092, LOCALHOST://localhost:9084 + KAFKA_BROKER_ID: 4 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + volumes: + - kafka-4:/bitnami/kafka + ports: + - 9084:9084 + - 7004:7000 + depends_on: + - zookeeper + + kafka-5: + build: kafka + environment: + KAFKA_ENABLE_KRAFT: 'false' + KAFKA_OPTS: + -javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.authenticate=false + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9085 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-5:9092, LOCALHOST://localhost:9085 + KAFKA_BROKER_ID: 5 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + volumes: + - kafka-5:/bitnami/kafka + ports: + - 9085:9085 + - 7005:7000 + depends_on: + - zookeeper + + setup: + image: juplo/toolbox + command: > + bash -c " + kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test + kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 33 --replication-factor 3 --config min.insync.replicas=2 + kafka-topics --bootstrap-server kafka:9092 --describe --topic test + " + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + - kafka-4 + - kafka-5 + + cli: + image: juplo/toolbox + command: sleep infinity + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + - kafka-4 + - kafka-5 + + burrow: + build: burrow + volumes: + - ./burrow.conf:/app/burrow.toml:ro + ports: + - 8080:8080 + + burrow_exporter: + build: burrow_exporter + environment: + BURROW_ADDR: http://burrow:8080 + API_VERSION: 3 + ports: + - 8008:8080 + + kafka-lag-exporter: + image: seglo/kafka-lag-exporter:0.8.2 + volumes: + - ./kafka-lag-exporter.conf:/opt/docker/conf/application.conf:ro + ports: + - 8000:8000 + + kafka-exporter: + image: danielqsj/kafka-exporter:v1.6.0 + command: --kafka.server kafka:9092 + ports: + - 9308:9308 + + prometheus: + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana:10.0.3 + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_USER: juplo + GF_SECURITY_ADMIN_PASSWORD: juplo + volumes: + - ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml + - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/automatic.yaml + - ./kafka-brokers.json:/var/lib/grafana/dashboards/main-dashboard.json + - grafana:/var/lib/grafana + + producer: + image: juplo/simple-producer:1.0-SNAPSHOT + entrypoint: >- + java + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.local.only=false + -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false + -jar /opt/app.jar + command: kafka:9092 test producer + ports: + - 7010:7000 + + consumer: + image: juplo/simple-consumer:1.0-SNAPSHOT + entrypoint: >- + java + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=7000 + -Dcom.sun.management.jmxremote.local.only=false + -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false + -jar /opt/app.jar + command: kafka:9092 test my-group consumer + ports: + - 7020:7000 + +volumes: + + zookeeper: + kafka-1: + kafka-2: + kafka-3: + kafka-4: + kafka-5: + grafana: