From 943222edc29da1d25f027cd3307d83ba046300b7 Mon Sep 17 00:00:00 2001
From: Kai Moritz <kai@juplo.de>
Date: Mon, 20 Feb 2023 21:01:00 +0100
Subject: [PATCH] Fixed the problem with the missing data in the Grfana
 dashboard

- The naming (and selection) of the broker-metrics that was defined in the
  configurtaion of the jmx-exporter did not match the naming in the
  predefined dashboard.
- Replaced the configuration for the jmx-exporter with the one that was
  defined in the same example, as the dashboard.
---
 kafka/jmx-exporter-config.yml | 313 ++++++++++++++++++++++++----------
 1 file changed, 227 insertions(+), 86 deletions(-)

diff --git a/kafka/jmx-exporter-config.yml b/kafka/jmx-exporter-config.yml
index 5f35f37..6ec2fe3 100644
--- a/kafka/jmx-exporter-config.yml
+++ b/kafka/jmx-exporter-config.yml
@@ -1,88 +1,229 @@
+startDelaySeconds: 120
 lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+cacheRules: true
+blacklistObjectNames:
+  - "kafka.consumer:type=*,id=*"
+  - "kafka.consumer:type=*,client-id=*"
+  - "kafka.consumer:type=*,client-id=*,node-id=*"
+  - "kafka.producer:type=*,id=*"
+  - "kafka.producer:type=*,client-id=*"
+  - "kafka.producer:type=*,client-id=*,node-id=*"
+  - "kafka.*:type=kafka-metrics-count,*"
+  # This will ignore the admin client metrics from Kafka Brokers and will blacklist certain metrics
+  # that do not make sense for ingestion.
+  # "kafka.admin.client:type=*, node-id=*, client-id=*"
+  # "kafka.admin.client:type=*, client-id=*"
+  # "kafka.admin.client:type=*, id=*"
+  - "kafka.admin.client:*"
+  - "kafka.server:type=*,cipher=*,protocol=*,listener=*,networkProcessor=*"
+  - "kafka.server:type=*"
+  - "kafka.server:type=app-info,id=*"
+  - "kafka.rest:*"
+  - "rest.utils:*"
+  - "io.confluent.common.security.jetty:*"
+  - "io.confluent.rest:*"
+  - "confluent.metadata.service:type=app-info,id=*"
+  - "confluent.metadata.service:type=app-info,client-id=*"
+  - "confluent.metadata:type=kafkaauthstore,*"
 rules:
-- pattern : kafka.cluster<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
-  name: kafka_cluster_$1_$2
-  labels:
-    topic: "$3"
-    partition: "$4"
-- pattern : kafka.log<type=Log, name=(.+), topic=(.+), partition=(.+)><>Value
-  name: kafka_log_$1
-  labels:
-    topic: "$2"
-    partition: "$3"
-- pattern : kafka.controller<type=(.+), name=(.+)><>(Count|Value)
-  name: kafka_controller_$1_$2
-- pattern : kafka.network<type=(.+), name=(.+)><>Value
-  name: kafka_network_$1_$2
-- pattern : kafka.network<type=(.+), name=(.+)PerSec, request=(.+)><>Count
-  name: kafka_network_$1_$2_total
-  labels:
-    request: "$3"
-- pattern : kafka.network<type=(.+), name=(\w+), networkProcessor=(.+)><>Count
-  name: kafka_network_$1_$2
-  labels:
-    request: "$3"
-  type: COUNTER
-- pattern : kafka.network<type=(.+), name=(\w+), request=(\w+)><>Count
-  name: kafka_network_$1_$2
-  labels:
-    request: "$3"
-- pattern : kafka.network<type=(.+), name=(\w+)><>Count
-  name: kafka_network_$1_$2
-- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
-  name: kafka_server_$1_$2_total
-  labels:
-    topic: "$3"
-- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*><>Count
-  name: kafka_server_$1_$2_total
-  type: COUNTER
-
-- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>(Count|Value)
-  name: kafka_server_$1_$2
-  labels:
-    clientId: "$3"
-    topic: "$4"
-    partition: "$5"
-- pattern : kafka.server<type=(.+), name=(.+), topic=(.+), partition=(.*)><>(Count|Value)
-  name: kafka_server_$1_$2
-  labels:
-    topic: "$3"
-    partition: "$4"
-- pattern : kafka.server<type=(.+), name=(.+), topic=(.+)><>(Count|Value)
-  name: kafka_server_$1_$2
-  labels:
-    topic: "$3"
-  type: COUNTER
-
-- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>(Count|Value)
-  name: kafka_server_$1_$2
-  labels:
-    clientId: "$3"
-    broker: "$4:$5"
-- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+)><>(Count|Value)
-  name: kafka_server_$1_$2
-  labels:
-    clientId: "$3"
-- pattern : kafka.server<type=(.+), name=(.+)><>(Count|Value)
-  name: kafka_server_$1_$2
-
-- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
-  name: kafka_$1_$2_$3_total
-- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
-  name: kafka_$1_$2_$3_total
-  labels:
-    topic: "$4"
-  type: COUNTER
-- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+), partition=(.+)><>Count
-  name: kafka_$1_$2_$3_total
-  labels:
-    topic: "$4"
-    partition: "$5"
-  type: COUNTER
-- pattern : kafka.(\w+)<type=(.+), name=(.+)><>(Count|Value)
-  name: kafka_$1_$2_$3_$4
-  type: COUNTER
-- pattern : kafka.(\w+)<type=(.+), name=(.+), (\w+)=(.+)><>(Count|Value)
-  name: kafka_$1_$2_$3_$6
-  labels:
-    "$4": "$5"
+  # This is by far the biggest contributor to the number of sheer metrics being produced.
+  # Always keep it on the top for the case of probability when so many metrics will hit the first condition and exit.
+  # "kafka.cluster:type=*, name=*, topic=*, partition=*"
+  # "kafka.log:type=*,name=*, topic=*, partition=*"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+    name: kafka_$1_$2_$3
+    type: GAUGE
+    labels:
+      topic: "$4"
+      partition: "$5"
+  # "kafka.server:type=*,name=*, client-id=*, topic=*, partition=*"
+  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+    name: kafka_server_$1_$2
+    type: GAUGE
+    labels:
+      clientId: "$3"
+      topic: "$4"
+      partition: "$5"
+  - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+    name: kafka_server_$1_$2
+    type: GAUGE
+    labels:
+      clientId: "$3"
+      broker: "$4:$5"
+  # "kafka.network:type=*, name=*, request=*, error=*"
+  # "kafka.network:type=*, name=*, request=*, version=*"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
+    name: kafka_$1_$2_$3
+    labels:
+      "$4": "$5"
+      "$6": "$7"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+    name: kafka_$1_$2_$3
+    type: GAUGE
+    labels:
+      "$4": "$5"
+      "$6": "$7"
+      quantile: "0.$8"
+  # "kafka.rest:type=*, topic=*, partition=*, client-id=*"
+  # "kafka.rest:type=*, cipher=*, protocol=*, client-id=*"
+  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>Value
+    name: kafka_$1_$2
+    labels:
+      "$3": "$4"
+      "$5": "$6"
+      "$7": "$8"
+  # Count and Value
+  # "kafka.server:type=*, name=*, topic=*"
+  # "kafka.server:type=*, name=*, clientId=*"
+  # "kafka.server:type=*, name=*, delayedOperation=*"
+  # "kafka.server:type=*, name=*, fetcherType=*"
+  # "kafka.network:type=*, name=*, networkProcessor=*"
+  # "kafka.network:type=*, name=*, processor=*"
+  # "kafka.network:type=*, name=*, request=*"
+  # "kafka.network:type=*, name=*, listener=*"
+  # "kafka.log:type=*, name=*, logDirectory=*"
+  # "kafka.log:type=*, name=*, op=*"
+  # "kafka.rest:type=*, node-id=*, client-id=*"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>(Count|Value)
+    name: kafka_$1_$2_$3
+    labels:
+      "$4": "$5"
+  # "kafka.consumer:type=*, topic=*, client-id=*"
+  # "kafka.producer:type=*, topic=*, client-id=*"
+  # "kafka.rest:type=*, topic=*, client-id=*"
+  # "kafka.server:type=*, broker-id=*, fetcher-id=*"
+  # "kafka.server:type=*, listener=*, networkProcessor=*"
+  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(Count|Value)
+    name: kafka_$1_$2
+    labels:
+      "$3": "$4"
+      "$5": "$6"
+  # - pattern: "kafka.(.+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+):"
+  #   name: kafka_$1_$2
+  #   labels:
+  #     "$3": "$4"
+  #     "$5": "$6"
+  #     attribute_name: "$7"
+  # "kafka.network:type=*, name=*"
+  # "kafka.server:type=*, name=*"
+  # "kafka.controller:type=*, name=*"
+  # "kafka.databalancer:type=*, name=*"
+  # "kafka.log:type=*, name=*"
+  # "kafka.utils:type=*, name=*"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(Count|Value)
+    name: kafka_$1_$2_$3
+  # "kafka.producer:type=*, client-id=*"
+  # "kafka.producer:type=*, id=*"
+  # "kafka.rest:type=*, client-id=*"
+  # "kafka.rest:type=*, http-status-code=*"
+  # "kafka.server:type=*, BrokerId=*"
+  # "kafka.server:type=*, listener=*"
+  # "kafka.server:type=*, id=*"
+  - pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>Value
+    name: kafka_$1_$2
+    labels:
+      "$3": "$4"
+  # - pattern: "kafka.(.+)<type=(.+), (.+)=(.+)><>(.+):"
+  #   name: kafka_$1_$2
+  #   labels:
+  #     "$3": "$4"
+  #     attribute_name: "$5"
+  - pattern: kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
+    name: kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total
+    type: GAUGE
+  # "kafka.server:type=*, listener=*, networkProcessor=*, clientSoftwareName=*, clientSoftwareVersion=*"
+  - pattern: kafka.server<type=socket-server-metrics, clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
+    name: kafka_server_socketservermetrics_connections
+    type: GAUGE
+    labels:
+      client_software_name: "$1"
+      client_software_version: "$2"
+      listener: "$3"
+      network_processor: "$4"
+  - pattern: "kafka.server<type=socket-server-metrics, listener=(.+), networkProcessor=(.+)><>(.+):"
+    name: kafka_server_socketservermetrics_$3
+    type: GAUGE
+    labels:
+      listener: "$1"
+      network_processor: "$2"
+  # - pattern: "kafka.server<type=socket-server-metrics, listener=(.+)><>(.+):"
+  #   name: kafka_server_socketservermetrics
+  #   type: GAUGE
+  #   labels:
+  #     listener: "$1"
+  #     attribute_name: "$2"
+  # "kafka.coordinator.group:type=*, name=*"
+  # "kafka.coordinator.transaction:type=*, name=*"
+  - pattern: kafka.coordinator.(\w+)<type=(.+), name=(.+)><>(Count|Value)
+    name: kafka_coordinator_$1_$2_$3
+  # Percentile
+  - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+    name: kafka_$1_$2_$3
+    type: GAUGE
+    labels:
+      "$4": "$5"
+      quantile: "0.$6"
+  - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+    name: kafka_$1_$2_$3
+    type: GAUGE
+    labels:
+      quantile: "0.$4"
+  # Additional Rules for Confluent Server Metrics
+  # 'confluent.metadata:type=*, name=*, topic=*, partition=*'
+  - pattern: confluent.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(Value|Count)
+    name: confluent_$1_$2
+    type: GAUGE
+    labels:
+      "$3": "$4"
+      "$5": "$6"
+      "$7": "$8"
+  # 'confluent.metadata.service:type=*, node-id=*, client-id=*'
+  - pattern: confluent.(.+)<type=(.+), (.+)=(.+), (.+)=(.+)><>Value
+    name: confluent_$1_$2
+    type: GAUGE
+    labels:
+      "$3": "$4"
+      "$5": "$6"
+  # 'confluent.metadata.service:type=*, node-id=*, client-id=*'
+  - pattern: 'confluent.metadata.service<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+):'
+    name: $1
+    type: GAUGE
+    labels:
+      "$2": "$3"
+      "$4": "$5"
+      attribute_name: "$6"
+  # 'confluent.metadata.service:type=*, client-id=*'
+  # 'confluent.metadata.service:type=*, id=*'
+  # 'confluent.metadata:type=*, name=*'
+  # 'confluent.license:type=*, name=*'
+  - pattern: confluent.(.+)<type=(.+), (.+)=(.+)><>Value
+    name: confluent_$1_$2
+    type: GAUGE
+    labels:
+      "$3": "$4"
+  - pattern: 'confluent.(.+)<type=(.+), (.+)=(.+)><>(.+):'
+    name: confluent_$1_$2
+    type: GAUGE
+    labels:
+      "$3": "$4"
+      attribute_name: "$5"
+  # Quotas
+  - pattern : 'kafka.server<type=(Produce|Fetch|Request), user=(.+), client-id=(.+)><>(.+):'
+    name: kafka_server_$1_$4
+    type: GAUGE
+    labels:
+      user: "$2"
+      client-id: "$3"
+  
+  - pattern : 'kafka.server<type=(Produce|Fetch|Request), user=(.+)><>(.+):'
+    name: kafka_server_$1_$3
+    type: GAUGE
+    labels:
+      user: "$2"
+  
+  - pattern : 'kafka.server<type=(Produce|Fetch|Request), client-id=(.+)><>(.+):'
+    name: kafka_server_$1_$3
+    type: GAUGE
+    labels:
+      client-id: "$2"
-- 
2.20.1