From 2e7d16fa6d050ccbb157e72e906f0f71e02005dc Mon Sep 17 00:00:00 2001
From: Kai Moritz <kai@juplo.de>
Date: Sun, 19 Feb 2023 10:26:41 +0100
Subject: [PATCH] Added `prometheus_jmx_exporter` as java-agent in the Kafka
 brokers

---
 docker-compose.yml            |  6 +--
 kafka/Dockerfile              |  9 ++++
 kafka/jmx-exporter-config.yml | 88 +++++++++++++++++++++++++++++++++++
 3 files changed, 100 insertions(+), 3 deletions(-)
 create mode 100644 kafka/Dockerfile
 create mode 100644 kafka/jmx-exporter-config.yml

diff --git a/docker-compose.yml b/docker-compose.yml
index 39a7b71..056be3c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,7 +6,7 @@ services:
       ALLOW_ANONYMOUS_LOGIN: 'yes'
 
   kafka-1:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081
@@ -23,7 +23,7 @@ services:
       - zookeeper
 
   kafka-2:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082
@@ -45,7 +45,7 @@ services:
       - zookeeper
 
   kafka-3:
-    image: bitnami/kafka:3.4.0
+    build: kafka
     environment:
       KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083
diff --git a/kafka/Dockerfile b/kafka/Dockerfile
new file mode 100644
index 0000000..7c6cc74
--- /dev/null
+++ b/kafka/Dockerfile
@@ -0,0 +1,9 @@
+FROM bitnami/kafka:3.4.0
+
+ADD jmx-exporter-config.yml /opt/prometheus/config.yml
+ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.17.2/jmx_prometheus_javaagent-0.17.2.jar /opt/prometheus/jmx_prometheus_javaagent.jar
+USER root
+RUN chmod -R 777 /opt/prometheus
+USER 1001
+
+ENV KAFKA_OPTS=-javaagent:/opt/prometheus/jmx_prometheus_javaagent.jar=80:/opt/prometheus/config.yml
diff --git a/kafka/jmx-exporter-config.yml b/kafka/jmx-exporter-config.yml
new file mode 100644
index 0000000..5f35f37
--- /dev/null
+++ b/kafka/jmx-exporter-config.yml
@@ -0,0 +1,88 @@
+lowercaseOutputName: true
+rules:
+- pattern : kafka.cluster<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+  name: kafka_cluster_$1_$2
+  labels:
+    topic: "$3"
+    partition: "$4"
+- pattern : kafka.log<type=Log, name=(.+), topic=(.+), partition=(.+)><>Value
+  name: kafka_log_$1
+  labels:
+    topic: "$2"
+    partition: "$3"
+- pattern : kafka.controller<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_controller_$1_$2
+- pattern : kafka.network<type=(.+), name=(.+)><>Value
+  name: kafka_network_$1_$2
+- pattern : kafka.network<type=(.+), name=(.+)PerSec, request=(.+)><>Count
+  name: kafka_network_$1_$2_total
+  labels:
+    request: "$3"
+- pattern : kafka.network<type=(.+), name=(\w+), networkProcessor=(.+)><>Count
+  name: kafka_network_$1_$2
+  labels:
+    request: "$3"
+  type: COUNTER
+- pattern : kafka.network<type=(.+), name=(\w+), request=(\w+)><>Count
+  name: kafka_network_$1_$2
+  labels:
+    request: "$3"
+- pattern : kafka.network<type=(.+), name=(\w+)><>Count
+  name: kafka_network_$1_$2
+- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
+  name: kafka_server_$1_$2_total
+  labels:
+    topic: "$3"
+- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*><>Count
+  name: kafka_server_$1_$2_total
+  type: COUNTER
+
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+    topic: "$4"
+    partition: "$5"
+- pattern : kafka.server<type=(.+), name=(.+), topic=(.+), partition=(.*)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    topic: "$3"
+    partition: "$4"
+- pattern : kafka.server<type=(.+), name=(.+), topic=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    topic: "$3"
+  type: COUNTER
+
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+    broker: "$4:$5"
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+- pattern : kafka.server<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+  name: kafka_$1_$2_$3_total
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  labels:
+    topic: "$4"
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+), partition=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  labels:
+    topic: "$4"
+    partition: "$5"
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_$1_$2_$3_$4
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+), (\w+)=(.+)><>(Count|Value)
+  name: kafka_$1_$2_$3_$6
+  labels:
+    "$4": "$5"
-- 
2.20.1