X-Git-Url: https://juplo.de/gitweb/?a=blobdiff_plain;f=kafka-brokers.json;h=13cbb5dc4a5e1442e156904e48015972921f0731;hb=1e602945d2ab1ac3c45c7de9f7499a53fd5a6033;hp=1dbbaafdd801aa39698495dcba2706f67011a8f2;hpb=4d3aee9eb9b903feaafe17a47f4ec3124c40b8de;p=demos%2Fkafka%2Fmonitoring diff --git a/kafka-brokers.json b/kafka-brokers.json index 1dbbaaf..13cbb5d 100644 --- a/kafka-brokers.json +++ b/kafka-brokers.json @@ -60,9 +60,12 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of active controllers in the cluster.", + "description": "Number of Brokers Online", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "mappings": [ { "options": { @@ -78,15 +81,16 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", + "color": "#d44a3a", "value": null }, { - "color": "#e5ac0e", - "value": 2 + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 }, { - "color": "#bf1b00" + "color": "#299c46", + "value": 2 } ] }, @@ -100,14 +104,14 @@ "x": 0, "y": 1 }, - "id": 12, + "id": 14, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", - "orientation": "vertical", + "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" @@ -116,25 +120,26 @@ "values": false }, "text": {}, - "textMode": "value_and_name" + "textMode": "auto" }, "pluginVersion": "10.0.3", + "repeatDirection": "h", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0", + "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "", "refId": "A" } ], - "title": "Active Controllers", + "title": "Brokers Online", "type": "stat" }, { @@ -142,12 +147,9 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of partitions under min insync replicas.", + "description": "Number of active controllers in the cluster.", "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, "mappings": [ { "options": { @@ -163,16 +165,15 @@ "mode": "absolute", "steps": [ { - "color": "#508642", + "color": "#299c46", "value": null }, { - "color": "rgba(237, 129, 40, 0.89)", - "value": 1 + "color": "#e5ac0e", + "value": 2 }, { - "color": "#bf1b00", - "value": 5 + "color": "#bf1b00" } ] }, @@ -182,18 +183,18 @@ }, "gridPos": { "h": 4, - "w": 4, + "w": 5, "x": 4, "y": 1 }, - "id": 32, + "id": 12, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", - "orientation": "horizontal", + "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" @@ -202,7 +203,7 @@ "values": false }, "text": {}, - "textMode": "auto" + "textMode": "value_and_name" }, "pluginVersion": "10.0.3", "targets": [ @@ -211,17 +212,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0", "format": "time_series", - "hide": false, "instant": true, "interval": "", - "intervalFactor": 2, - "legendFormat": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", "refId": "A" } ], - "title": "Under Min ISR Partitions", + "title": "Active Controllers", "type": "stat" }, { @@ -229,7 +229,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of partitions that dont have an active leader and are hence not writable or readable.", + "description": "Partitions that are online", "fieldConfig": { "defaults": { "color": { @@ -250,16 +250,16 @@ "mode": "absolute", "steps": [ { - "color": "#508642", + "color": "#d44a3a", "value": null }, { - "color": "#ef843c", - "value": 1 + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 }, { - "color": "#bf1b00", - "value": 1 + "color": "#299c46", + "value": 0 } ] }, @@ -269,11 +269,11 @@ }, "gridPos": { "h": 4, - "w": 4, - "x": 8, + "w": 6, + "x": 9, "y": 1 }, - "id": 22, + "id": 18, "links": [], "maxDataPoints": 100, "options": { @@ -298,7 +298,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", @@ -307,7 +307,7 @@ "refId": "A" } ], - "title": "Offline Partitions Count", + "title": "Online Partitions", "type": "stat" }, { @@ -355,7 +355,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 12, + "x": 15, "y": 1 }, "id": 33, @@ -400,125 +400,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 84, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Bytes in", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Bytes out", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "B", - "step": 4 - } - ], - "title": "Broker network throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Number of under-replicated partitions (| ISR | < | all replicas |).", + "description": "Unclean leader election rate", "fieldConfig": { "defaults": { "color": { @@ -539,16 +421,15 @@ "mode": "absolute", "steps": [ { - "color": "#508642", + "color": "#299c46", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", - "value": 1 + "value": 2 }, { - "color": "#bf1b00", - "value": 5 + "color": "#d44a3a" } ] }, @@ -559,10 +440,10 @@ "gridPos": { "h": 4, "w": 4, - "x": 0, - "y": 5 + "x": 19, + "y": 1 }, - "id": 20, + "id": 16, "links": [], "maxDataPoints": 100, "options": { @@ -587,17 +468,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", "format": "time_series", - "hide": false, "instant": true, "interval": "", - "intervalFactor": 2, + "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], - "title": "Under Replicated Partitions", + "title": "Unclean Leader Election Rate", "type": "stat" }, { @@ -605,7 +485,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of Brokers Online", + "description": "Number of under-replicated partitions (| ISR | < | all replicas |).", "fieldConfig": { "defaults": { "color": { @@ -626,16 +506,16 @@ "mode": "absolute", "steps": [ { - "color": "#d44a3a", + "color": "#508642", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", - "value": 0 + "value": 1 }, { - "color": "#299c46", - "value": 2 + "color": "#bf1b00", + "value": 5 } ] }, @@ -646,10 +526,10 @@ "gridPos": { "h": 4, "w": 4, - "x": 4, + "x": 0, "y": 5 }, - "id": 14, + "id": 20, "links": [], "maxDataPoints": 100, "options": { @@ -668,23 +548,23 @@ "textMode": "auto" }, "pluginVersion": "10.0.3", - "repeatDirection": "h", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", "format": "time_series", + "hide": false, "instant": true, "interval": "", - "intervalFactor": 1, + "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], - "title": "Brokers Online", + "title": "Under Replicated Partitions", "type": "stat" }, { @@ -692,178 +572,111 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Partitions that are online", + "description": "", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#d44a3a", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Online Partitions", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Unclean leader election rate", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "#299c46", + "color": "green", "value": null }, { - "color": "rgba(237, 129, 40, 0.89)", - "value": 2 - }, - { - "color": "#d44a3a" + "color": "red", + "value": 80 } ] }, - "unit": "none" + "unit": "partitions" }, "overrides": [] }, "gridPos": { - "h": 4, - "w": 4, - "x": 12, + "h": 12, + "w": 11, + "x": 4, "y": 5 }, - "id": 16, - "links": [], - "maxDataPoints": 100, + "id": 123, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { + "legend": { "calcs": [ - "lastNotNull" + "lastNotNull", + "max" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "text": {}, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "10.0.3", + "pluginVersion": "8.1.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", - "format": "time_series", - "instant": true, + "editorMode": "code", + "exemplar": false, + "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], - "title": "Unclean Leader Election Rate", - "type": "stat" + "title": "Underreplicated Partitions", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -899,8 +712,19 @@ "mode": "off" } }, + "decimals": 0, "links": [], - "mappings": [], + "mappings": [ + { + "options": { + "null": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "min": 0, "thresholds": { "mode": "absolute", @@ -915,46 +739,20 @@ } ] }, - "unit": "ops" + "unit": "partitions" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "kafka-3:80" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 + "h": 12, + "w": 8, + "x": 15, + "y": 5 }, - "id": 123, + "id": 124, "options": { "legend": { "calcs": [ - "mean", "lastNotNull", "max" ], @@ -976,7 +774,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", + "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", "hide": false, "interval": "", "legendFormat": "{{instance}}", @@ -984,43 +782,20 @@ "refId": "A" } ], - "title": "IsrExpands per Sec", + "title": "Leader Count", "type": "timeseries" }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 31, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Request rate", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Produce request rate.", + "description": "Number of partitions under min insync replicas.", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "mappings": [ { "options": { @@ -1036,8 +811,16 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "#508642", "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#bf1b00", + "value": 5 } ] }, @@ -1049,14 +832,14 @@ "h": 4, "w": 4, "x": 0, - "y": 19 + "y": 9 }, - "id": 93, + "id": 32, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1076,13 +859,17 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))", + "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "instant": true, "interval": "", + "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], - "title": "All Request Per Sec", + "title": "Under Min ISR Partitions", "type": "stat" }, { @@ -1090,9 +877,12 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Produce request rate.", + "description": "Number of partitions that dont have an active leader and are hence not writable or readable.", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "mappings": [ { "options": { @@ -1108,8 +898,16 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "#508642", "value": null + }, + { + "color": "#ef843c", + "value": 1 + }, + { + "color": "#bf1b00", + "value": 1 } ] }, @@ -1120,15 +918,15 @@ "gridPos": { "h": 4, "w": 4, - "x": 4, - "y": 19 + "x": 0, + "y": 13 }, - "id": 35, + "id": 22, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { @@ -1148,302 +946,430 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))", + "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "format": "time_series", + "instant": true, "interval": "", + "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], - "title": "Produce Request Per Sec", + "title": "Offline Partitions Count", "type": "stat" }, { + "collapsed": true, "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Fetch request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] + "type": "datasource", + "uid": "grafana" }, "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 19 - }, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" + "h": 1, + "w": 24, + "x": 0, + "y": 17 }, - "pluginVersion": "10.0.3", - "targets": [ + "id": 40, + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Consumer Fetch Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Fetch request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Cores", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ { - "color": "green", - "value": null + "matcher": { + "id": "byName", + "options": "localhost:7071" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] } ] }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 23 - }, - "id": 94, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 27, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "process_cpu_secondspersec", + "refId": "A", + "step": 4 + } ], - "fields": "", - "values": false + "title": "CPU Usage", + "type": "timeseries" }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Broker Fetch Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Offset Commit request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ { - "color": "green", - "value": null + "matcher": { + "id": "byName", + "options": "localhost:7071" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "kafka-2:80" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] } ] }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 23 - }, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "jvm_memory_bytes_used", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "B" + } ], - "fields": "", - "values": false + "title": "JVM Memory Used", + "type": "timeseries" }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Offset Commit Request Per Sec", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Metadata request rate.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "% time in GC", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ { - "color": "green", - "value": null + "matcher": { + "id": "byName", + "options": "localhost:7071" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] } ] }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 23 - }, - "id": 36, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "jvm_gc_collection_seconds_sum", + "refId": "A", + "step": 4 + } + ], + "title": "Time spent in GC", + "type": "timeseries" + } + ], "targets": [ { "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "type": "datasource", + "uid": "grafana" }, - "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))", - "interval": "", - "legendFormat": "", "refId": "A" } ], - "title": "Metadata Request Per Sec", - "type": "stat" + "title": "System", + "type": "row" }, { "collapsed": false, @@ -1455,9 +1381,9 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 18 }, - "id": 40, + "id": 31, "panels": [], "targets": [ { @@ -1468,7 +1394,7 @@ "refId": "A" } ], - "title": "System", + "title": "Throughput & Request rate", "type": "row" }, { @@ -1484,7 +1410,7 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Cores", + "axisLabel": "Bytes/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1526,33 +1452,17 @@ } ] }, - "unit": "percent" + "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, - "y": 28 + "y": 19 }, - "id": 27, + "id": 84, "links": [], "options": { "legend": { @@ -1561,7 +1471,7 @@ "lastNotNull", "max" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -1577,17 +1487,32 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100", + "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", "format": "time_series", "interval": "", "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "process_cpu_secondspersec", + "legendFormat": "Bytes in", + "metric": "kafka_server_brokertopicmetrics_bytesinpersec", "refId": "A", "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Bytes out", + "metric": "kafka_server_brokertopicmetrics_bytesinpersec", + "refId": "B", + "step": 4 } ], - "title": "CPU Usage", + "title": "Broker network throughput", "type": "timeseries" }, { @@ -1595,1595 +1520,1298 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Produce request rate.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Memory", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - }, - "links": [], - "mappings": [], + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 28 + "h": 4, + "w": 4, + "x": 12, + "y": 19 }, - "id": 2, + "id": 93, "links": [], + "maxDataPoints": 100, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { "calcs": [ - "mean", - "lastNotNull", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "text": {}, + "textMode": "auto" }, - "pluginVersion": "8.1.3", + "pluginVersion": "10.0.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "jvm_memory_bytes_used", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}", + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))", "interval": "", - "legendFormat": "{{instance}}", - "refId": "B" + "legendFormat": "", + "refId": "A" } ], - "title": "JVM Memory Used", - "type": "timeseries" + "title": "All Request Per Sec", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Fetch request rate.", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "% time in GC", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" } - }, - "links": [], - "mappings": [], + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percentunit" + "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "localhost:7071" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, + "h": 4, + "w": 4, "x": 16, - "y": 28 + "y": 19 }, - "id": 3, + "id": 37, "links": [], + "maxDataPoints": 100, "options": { - "legend": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { "calcs": [ - "mean", - "lastNotNull", - "max" + "lastNotNull" ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "text": {}, + "textMode": "auto" }, - "pluginVersion": "8.1.3", + "pluginVersion": "10.0.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "jvm_gc_collection_seconds_sum", - "refId": "A", - "step": 4 + "legendFormat": "", + "refId": "A" } ], - "title": "Time spent in GC", - "type": "timeseries" + "title": "Consumer Fetch Request Per Sec", + "type": "stat" }, { - "collapsed": true, "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Produce request rate.", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 + "h": 4, + "w": 4, + "x": 20, + "y": 19 }, - "id": 29, - "panels": [ + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Messages/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Produce Request Per Sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Metadata request rate.", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 4, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single" + "type": "special" } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "bytes/sec", - "metric": "kafka_server_brokertopicmetrics_messagesinpersec", - "refId": "A", - "step": 4 - } + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 23 + }, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Messages In", - "type": "timeseries" + "fields": "", + "values": false }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 24 - }, - "id": 5, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.3", - "targets": [ + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Metadata Request Per Sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Offset Commit request rate.", + "fieldConfig": { + "defaults": { + "mappings": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "bytes/sec", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 - } - ], - "title": "Bytes In", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "options": { + "match": "null", + "result": { + "text": "N/A" } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 24 - }, - "id": 6, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "exemplar": true, - "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "bytes/sec", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 + "type": "special" } ], - "title": "Bytes Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Messages/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 32 - }, - "id": 10, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single" - } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "kafka_server_brokertopicmetrics_messagesinpersec", - "refId": "A", - "step": 4 - } + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 23 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Messages In Per Broker", - "type": "timeseries" + "fields": "", + "values": false }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bytes/s", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 32 - }, - "id": 7, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.3", - "targets": [ + }, + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Offset Commit Request Per Sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Fetch request rate.", + "fieldConfig": { + "defaults": { + "mappings": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "options": { + "match": "null", + "result": { + "text": "N/A" + } }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "kafka_server_brokertopicmetrics_bytesinpersec", - "refId": "A", - "step": 4 + "type": "special" } ], - "title": "Bytes In Per Broker", - "type": "timeseries" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 23 + }, + "id": 94, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.0.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] + "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Broker Fetch Request Per Sec", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 29, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 32 + "refId": "A" + } + ], + "title": "Throughput In/Out", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Messages/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 4, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Bytes Out Per Broker", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], + }, + "pluginVersion": "8.1.3", "targets": [ { "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "refId": "A" + "exemplar": true, + "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "bytes/sec", + "metric": "kafka_server_brokertopicmetrics_messagesinpersec", + "refId": "A", + "step": 4 } ], - "title": "Throughput In/Out", - "type": "row" + "title": "Messages In", + "type": "timeseries" }, { - "collapsed": true, "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "id": 44, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 25 + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Network Processor Avg Usage Percent", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 5, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Request Handler Avg Percent", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], + }, + "pluginVersion": "8.1.3", "targets": [ { "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "refId": "A" + "exemplar": true, + "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "bytes/sec", + "metric": "kafka_server_brokertopicmetrics_bytesinpersec", + "refId": "A", + "step": 4 } ], - "title": "Thread utilization", - "type": "row" + "title": "Bytes In", + "type": "timeseries" }, { - "collapsed": true, "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 + "h": 8, + "w": 8, + "x": 16, + "y": 28 }, - "id": 86, - "panels": [ + "id": 6, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency in millseconds for ZooKeeper requests from broker.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 26 + "exemplar": true, + "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "bytes/sec", + "metric": "kafka_server_brokertopicmetrics_bytesinpersec", + "refId": "A", + "step": 4 + } + ], + "title": "Bytes Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 88, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Messages/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}", - "hide": false, - "legendFormat": "{{instance}} - {{quantile}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 10, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Zookeeper Request Latency", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 26 + "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "kafka_server_brokertopicmetrics_messagesinpersec", + "refId": "A", + "step": 4 + } + ], + "title": "Messages In Per Broker", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 92, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Zookeeper connections per sec", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 35 + "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "kafka_server_brokertopicmetrics_bytesinpersec", + "refId": "A", + "step": 4 + } + ], + "title": "Bytes In Per Broker", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 89, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Zookeeper expired connections per sec", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] + "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Bytes Out Per Broker", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 44, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 35 + "refId": "A" + } + ], + "title": "Thread utilization", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 90, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Zookeeper disconnect per sec", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 35 + "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Network Processor Avg Usage Percent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 91, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" ], - "title": "Zookeeper auth failures per sec", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], + }, + "pluginVersion": "8.1.3", "targets": [ { "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, + "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{instance}}", "refId": "A" } ], - "title": "Zookeeper", - "type": "row" + "title": "Request Handler Avg Percent", + "type": "timeseries" }, { "collapsed": false, @@ -3195,7 +2823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 57 }, "id": 82, "panels": [], @@ -3259,8 +2887,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3270,13 +2897,38 @@ }, "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "kafka-2:80" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 40 + "y": 58 }, "id": 80, "options": { @@ -3359,8 +3011,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3370,13 +3021,38 @@ }, "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "kafka-2:80" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 40 + "y": 58 }, "id": 83, "options": { @@ -3422,7 +3098,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 67 }, "id": 53, "panels": [ @@ -3635,7 +3311,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 68 }, "id": 58, "panels": [ @@ -4138,7 +3814,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 69 }, "id": 68, "panels": [ @@ -4643,7 +4319,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 70 }, "id": 66, "panels": [ @@ -5148,7 +4824,7 @@ "h": 1, "w": 24, "x": 0, - "y": 53 + "y": 71 }, "id": 102, "panels": [ @@ -5929,7 +5605,7 @@ "h": 1, "w": 24, "x": 0, - "y": 54 + "y": 72 }, "id": 120, "panels": [ @@ -6188,7 +5864,7 @@ "h": 1, "w": 24, "x": 0, - "y": 55 + "y": 73 }, "id": 46, "panels": [ @@ -6469,7 +6145,7 @@ "type": "row" } ], - "refresh": "5s", + "refresh": "", "schemaVersion": 38, "style": "dark", "tags": [], @@ -6577,7 +6253,7 @@ ] }, "time": { - "from": "now-12h", + "from": "now-5m", "to": "now" }, "timepicker": {