From 943222edc29da1d25f027cd3307d83ba046300b7 Mon Sep 17 00:00:00 2001 From: Kai Moritz Date: Mon, 20 Feb 2023 21:01:00 +0100 Subject: [PATCH] Fixed the problem with the missing data in the Grfana dashboard - The naming (and selection) of the broker-metrics that was defined in the configurtaion of the jmx-exporter did not match the naming in the predefined dashboard. - Replaced the configuration for the jmx-exporter with the one that was defined in the same example, as the dashboard. --- kafka/jmx-exporter-config.yml | 313 ++++++++++++++++++++++++---------- 1 file changed, 227 insertions(+), 86 deletions(-) diff --git a/kafka/jmx-exporter-config.yml b/kafka/jmx-exporter-config.yml index 5f35f37..6ec2fe3 100644 --- a/kafka/jmx-exporter-config.yml +++ b/kafka/jmx-exporter-config.yml @@ -1,88 +1,229 @@ +startDelaySeconds: 120 lowercaseOutputName: true +lowercaseOutputLabelNames: true +cacheRules: true +blacklistObjectNames: + - "kafka.consumer:type=*,id=*" + - "kafka.consumer:type=*,client-id=*" + - "kafka.consumer:type=*,client-id=*,node-id=*" + - "kafka.producer:type=*,id=*" + - "kafka.producer:type=*,client-id=*" + - "kafka.producer:type=*,client-id=*,node-id=*" + - "kafka.*:type=kafka-metrics-count,*" + # This will ignore the admin client metrics from Kafka Brokers and will blacklist certain metrics + # that do not make sense for ingestion. + # "kafka.admin.client:type=*, node-id=*, client-id=*" + # "kafka.admin.client:type=*, client-id=*" + # "kafka.admin.client:type=*, id=*" + - "kafka.admin.client:*" + - "kafka.server:type=*,cipher=*,protocol=*,listener=*,networkProcessor=*" + - "kafka.server:type=*" + - "kafka.server:type=app-info,id=*" + - "kafka.rest:*" + - "rest.utils:*" + - "io.confluent.common.security.jetty:*" + - "io.confluent.rest:*" + - "confluent.metadata.service:type=app-info,id=*" + - "confluent.metadata.service:type=app-info,client-id=*" + - "confluent.metadata:type=kafkaauthstore,*" rules: -- pattern : kafka.cluster<>Value - name: kafka_cluster_$1_$2 - labels: - topic: "$3" - partition: "$4" -- pattern : kafka.log<>Value - name: kafka_log_$1 - labels: - topic: "$2" - partition: "$3" -- pattern : kafka.controller<>(Count|Value) - name: kafka_controller_$1_$2 -- pattern : kafka.network<>Value - name: kafka_network_$1_$2 -- pattern : kafka.network<>Count - name: kafka_network_$1_$2_total - labels: - request: "$3" -- pattern : kafka.network<>Count - name: kafka_network_$1_$2 - labels: - request: "$3" - type: COUNTER -- pattern : kafka.network<>Count - name: kafka_network_$1_$2 - labels: - request: "$3" -- pattern : kafka.network<>Count - name: kafka_network_$1_$2 -- pattern : kafka.server<>Count - name: kafka_server_$1_$2_total - labels: - topic: "$3" -- pattern : kafka.server<>Count - name: kafka_server_$1_$2_total - type: COUNTER - -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - labels: - clientId: "$3" - topic: "$4" - partition: "$5" -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - labels: - topic: "$3" - partition: "$4" -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - labels: - topic: "$3" - type: COUNTER - -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - labels: - clientId: "$3" - broker: "$4:$5" -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - labels: - clientId: "$3" -- pattern : kafka.server<>(Count|Value) - name: kafka_server_$1_$2 - -- pattern : kafka.(\w+)<>Count - name: kafka_$1_$2_$3_total -- pattern : kafka.(\w+)<>Count - name: kafka_$1_$2_$3_total - labels: - topic: "$4" - type: COUNTER -- pattern : kafka.(\w+)<>Count - name: kafka_$1_$2_$3_total - labels: - topic: "$4" - partition: "$5" - type: COUNTER -- pattern : kafka.(\w+)<>(Count|Value) - name: kafka_$1_$2_$3_$4 - type: COUNTER -- pattern : kafka.(\w+)<>(Count|Value) - name: kafka_$1_$2_$3_$6 - labels: - "$4": "$5" + # This is by far the biggest contributor to the number of sheer metrics being produced. + # Always keep it on the top for the case of probability when so many metrics will hit the first condition and exit. + # "kafka.cluster:type=*, name=*, topic=*, partition=*" + # "kafka.log:type=*,name=*, topic=*, partition=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + topic: "$4" + partition: "$5" + # "kafka.server:type=*,name=*, client-id=*, topic=*, partition=*" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + # "kafka.network:type=*, name=*, request=*, error=*" + # "kafka.network:type=*, name=*, request=*, version=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + # "kafka.rest:type=*, topic=*, partition=*, client-id=*" + # "kafka.rest:type=*, cipher=*, protocol=*, client-id=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2 + labels: + "$3": "$4" + "$5": "$6" + "$7": "$8" + # Count and Value + # "kafka.server:type=*, name=*, topic=*" + # "kafka.server:type=*, name=*, clientId=*" + # "kafka.server:type=*, name=*, delayedOperation=*" + # "kafka.server:type=*, name=*, fetcherType=*" + # "kafka.network:type=*, name=*, networkProcessor=*" + # "kafka.network:type=*, name=*, processor=*" + # "kafka.network:type=*, name=*, request=*" + # "kafka.network:type=*, name=*, listener=*" + # "kafka.log:type=*, name=*, logDirectory=*" + # "kafka.log:type=*, name=*, op=*" + # "kafka.rest:type=*, node-id=*, client-id=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + # "kafka.consumer:type=*, topic=*, client-id=*" + # "kafka.producer:type=*, topic=*, client-id=*" + # "kafka.rest:type=*, topic=*, client-id=*" + # "kafka.server:type=*, broker-id=*, fetcher-id=*" + # "kafka.server:type=*, listener=*, networkProcessor=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2 + labels: + "$3": "$4" + "$5": "$6" + # - pattern: "kafka.(.+)<>(.+):" + # name: kafka_$1_$2 + # labels: + # "$3": "$4" + # "$5": "$6" + # attribute_name: "$7" + # "kafka.network:type=*, name=*" + # "kafka.server:type=*, name=*" + # "kafka.controller:type=*, name=*" + # "kafka.databalancer:type=*, name=*" + # "kafka.log:type=*, name=*" + # "kafka.utils:type=*, name=*" + - pattern: kafka.(\w+)<>(Count|Value) + name: kafka_$1_$2_$3 + # "kafka.producer:type=*, client-id=*" + # "kafka.producer:type=*, id=*" + # "kafka.rest:type=*, client-id=*" + # "kafka.rest:type=*, http-status-code=*" + # "kafka.server:type=*, BrokerId=*" + # "kafka.server:type=*, listener=*" + # "kafka.server:type=*, id=*" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2 + labels: + "$3": "$4" + # - pattern: "kafka.(.+)<>(.+):" + # name: kafka_$1_$2 + # labels: + # "$3": "$4" + # attribute_name: "$5" + - pattern: kafka.server<>OneMinuteRate + name: kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total + type: GAUGE + # "kafka.server:type=*, listener=*, networkProcessor=*, clientSoftwareName=*, clientSoftwareVersion=*" + - pattern: kafka.server<>connections + name: kafka_server_socketservermetrics_connections + type: GAUGE + labels: + client_software_name: "$1" + client_software_version: "$2" + listener: "$3" + network_processor: "$4" + - pattern: "kafka.server<>(.+):" + name: kafka_server_socketservermetrics_$3 + type: GAUGE + labels: + listener: "$1" + network_processor: "$2" + # - pattern: "kafka.server<>(.+):" + # name: kafka_server_socketservermetrics + # type: GAUGE + # labels: + # listener: "$1" + # attribute_name: "$2" + # "kafka.coordinator.group:type=*, name=*" + # "kafka.coordinator.transaction:type=*, name=*" + - pattern: kafka.coordinator.(\w+)<>(Count|Value) + name: kafka_coordinator_$1_$2_$3 + # Percentile + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" + # Additional Rules for Confluent Server Metrics + # 'confluent.metadata:type=*, name=*, topic=*, partition=*' + - pattern: confluent.(\w+)<>(Value|Count) + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + "$5": "$6" + "$7": "$8" + # 'confluent.metadata.service:type=*, node-id=*, client-id=*' + - pattern: confluent.(.+)<>Value + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + "$5": "$6" + # 'confluent.metadata.service:type=*, node-id=*, client-id=*' + - pattern: 'confluent.metadata.service<>(.+):' + name: $1 + type: GAUGE + labels: + "$2": "$3" + "$4": "$5" + attribute_name: "$6" + # 'confluent.metadata.service:type=*, client-id=*' + # 'confluent.metadata.service:type=*, id=*' + # 'confluent.metadata:type=*, name=*' + # 'confluent.license:type=*, name=*' + - pattern: confluent.(.+)<>Value + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + - pattern: 'confluent.(.+)<>(.+):' + name: confluent_$1_$2 + type: GAUGE + labels: + "$3": "$4" + attribute_name: "$5" + # Quotas + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$4 + type: GAUGE + labels: + user: "$2" + client-id: "$3" + + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$3 + type: GAUGE + labels: + user: "$2" + + - pattern : 'kafka.server<>(.+):' + name: kafka_server_$1_$3 + type: GAUGE + labels: + client-id: "$2" -- 2.20.1