From: Kai Moritz Date: Tue, 3 Oct 2023 22:01:32 +0000 (+0200) Subject: WIP:prometheus X-Git-Url: http://juplo.de/gitweb/?a=commitdiff_plain;h=6f6f2a8693a9aceb44ce82527d489ec384496ed3;p=demos%2Fkafka%2Fmonitoring WIP:prometheus --- diff --git a/strimzi-kafka/cluster.yaml b/strimzi-kafka/cluster.yaml index 792223f..9c10072 100644 --- a/strimzi-kafka/cluster.yaml +++ b/strimzi-kafka/cluster.yaml @@ -30,6 +30,12 @@ spec: class: local-storage size: 1Gi deleteClaim: false + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: kafka-metrics-config.yml zookeeper: replicas: 1 storage: @@ -38,7 +44,19 @@ spec: class: local-storage size: 500Mi deleteClaim: false + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: kafka-metrics + key: zookeeper-metrics-config.yml entityOperator: topicOperator: {} - cruiseControl: {} + cruiseControl: + metricsConfig: + type: jmxPrometheusExporter + valueFrom: + configMapKeyRef: + name: cruise-control-metrics + key: metrics-config.yml kafkaExporter: {} diff --git a/strimzi-kafka/kafka-cruise-control-metrics.yaml b/strimzi-kafka/kafka-cruise-control-metrics.yaml new file mode 100644 index 0000000..08c3a80 --- /dev/null +++ b/strimzi-kafka/kafka-cruise-control-metrics.yaml @@ -0,0 +1,14 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: cruise-control-metrics + labels: + app: strimzi +data: + metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + - pattern: kafka.cruisecontrol<>(\w+) + name: kafka_cruisecontrol_$1_$2 + type: GAUGE diff --git a/strimzi-kafka/kafka-metrics.yaml b/strimzi-kafka/kafka-metrics.yaml new file mode 100644 index 0000000..979d60b --- /dev/null +++ b/strimzi-kafka/kafka-metrics.yaml @@ -0,0 +1,183 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kafka-metrics + labels: + app: strimzi +data: + kafka-metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + # Special cases and very specific rules + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern: kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_tls_info + type: GAUGE + labels: + cipher: "$2" + protocol: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: kafka.server<>connections + name: kafka_server_$1_connections_software + type: GAUGE + labels: + clientSoftwareName: "$2" + clientSoftwareVersion: "$3" + listener: "$4" + networkProcessor: "$5" + - pattern: "kafka.server<>(.+):" + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + - pattern: kafka.server<>(.+) + name: kafka_server_$1_$4 + type: GAUGE + labels: + listener: "$2" + networkProcessor: "$3" + # Some percent metrics use MeanRate attribute + # Ex) kafka.server<>MeanRate + - pattern: kafka.(\w+)<>MeanRate + name: kafka_$1_$2_$3_percent + type: GAUGE + # Generic gauges for percents + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3_percent + type: GAUGE + labels: + "$4": "$5" + # Generic per-second counters with 0-2 key/value pairs + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + # Generic gauges with 0-2 key/value pairs + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. + # Note that these are missing the '_sum' metric! + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + - pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" + # KRaft mode: uncomment the following lines to export KRaft related metrics + # KRaft overall related metrics + # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics + #- pattern: "kafka.server<>(.+-total|.+-max):" + # name: kafka_server_raftmetrics_$1 + # type: COUNTER + #- pattern: "kafka.server<>(.+):" + # name: kafka_server_raftmetrics_$1 + # type: GAUGE + # KRaft "low level" channels related metrics + # distinguish between always increasing COUNTER (total and max) and variable GAUGE (all others) metrics + #- pattern: "kafka.server<>(.+-total|.+-max):" + # name: kafka_server_raftchannelmetrics_$1 + # type: COUNTER + #- pattern: "kafka.server<>(.+):" + # name: kafka_server_raftchannelmetrics_$1 + # type: GAUGE + # Broker metrics related to fetching metadata topic records in KRaft mode + #- pattern: "kafka.server<>(.+):" + # name: kafka_server_brokermetadatametrics_$1 + # type: GAUGE + zookeeper-metrics-config.yml: | + # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics + lowercaseOutputName: true + rules: + # replicated Zookeeper + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$2" + type: GAUGE + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$3" + type: GAUGE + labels: + replicaId: "$2" + - pattern: "org.apache.ZooKeeperService<>(Packets\\w+)" + name: "zookeeper_$4" + type: COUNTER + labels: + replicaId: "$2" + memberType: "$3" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4" + type: GAUGE + labels: + replicaId: "$2" + memberType: "$3" + - pattern: "org.apache.ZooKeeperService<>(\\w+)" + name: "zookeeper_$4_$5" + type: GAUGE + labels: + replicaId: "$2" + memberType: "$3"