"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of partitions under min insync replicas.",
+ "description": "Number of Brokers Online",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#508642",
+ "color": "#d44a3a",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
- "value": 1
+ "value": 0
},
{
- "color": "#bf1b00",
- "value": 5
+ "color": "#299c46",
+ "value": 2
}
]
},
"x": 4,
"y": 1
},
- "id": 32,
+ "id": 14,
"links": [],
"maxDataPoints": 100,
"options": {
"textMode": "auto"
},
"pluginVersion": "10.0.3",
+ "repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
- "hide": false,
"instant": true,
"interval": "",
- "intervalFactor": 2,
+ "intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
- "title": "Under Min ISR Partitions",
+ "title": "Brokers Online",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
+ "description": "Unclean leader election rate",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#508642",
+ "color": "#299c46",
"value": null
},
{
- "color": "#ef843c",
- "value": 1
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 2
},
{
- "color": "#bf1b00",
- "value": 1
+ "color": "#d44a3a"
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
+ "x": 11,
"y": 1
},
- "id": 22,
+ "id": 16,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"instant": true,
"interval": "",
"refId": "A"
}
],
- "title": "Offline Partitions Count",
+ "title": "Unclean Leader Election Rate",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
+ "description": "Partitions that are online",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#299c46",
+ "color": "#d44a3a",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
- "value": 2
+ "value": 0
},
{
- "color": "#d44a3a"
+ "color": "#299c46",
+ "value": 0
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 12,
+ "x": 15,
"y": 1
},
- "id": 33,
+ "id": 18,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"instant": true,
"interval": "",
"refId": "A"
}
],
- "title": "Preferred Replica Imbalance",
+ "title": "Online Partitions",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "palette-classic"
+ "mode": "thresholds"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
}
- },
- "links": [],
- "mappings": [],
+ ],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "green",
+ "color": "#299c46",
"value": null
},
{
- "color": "red",
- "value": 80
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 2
+ },
+ {
+ "color": "#d44a3a"
}
]
},
- "unit": "Bps"
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
+ "h": 4,
+ "w": 4,
+ "x": 19,
"y": 1
},
- "id": 84,
+ "id": 33,
"links": [],
+ "maxDataPoints": 100,
"options": {
- "legend": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
"calcs": [
- "mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "fields": "",
+ "values": false
},
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
+ "text": {},
+ "textMode": "auto"
},
- "pluginVersion": "8.1.3",
+ "pluginVersion": "10.0.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Bytes in",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
- "hide": false,
+ "instant": true,
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "Bytes out",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "B",
- "step": 4
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
}
],
- "title": "Broker network throughput",
- "type": "timeseries"
+ "title": "Preferred Replica Imbalance",
+ "type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of Brokers Online",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
+ },
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "#d44a3a",
+ "color": "green",
"value": null
},
{
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0
- },
- {
- "color": "#299c46",
- "value": 2
+ "color": "red",
+ "value": 80
}
]
},
- "unit": "none"
+ "unit": "partitions"
},
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 4,
+ "h": 12,
+ "w": 11,
"x": 4,
"y": 5
},
- "id": 14,
- "links": [],
- "maxDataPoints": 100,
+ "id": 123,
"options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
+ "legend": {
"calcs": [
- "lastNotNull"
+ "lastNotNull",
+ "max"
],
- "fields": "",
- "values": false
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
- "text": {},
- "textMode": "auto"
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "pluginVersion": "10.0.3",
- "repeatDirection": "h",
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "hide": false,
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
+ "legendFormat": "{{instance}}",
+ "range": true,
"refId": "A"
}
],
- "title": "Brokers Online",
- "type": "stat"
+ "title": "Underreplicated Partitions",
+ "type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Partitions that are online",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "partitions"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 15,
+ "y": 5
+ },
+ "id": 124,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Leader Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Number of partitions under min insync replicas.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#d44a3a",
+ "color": "#508642",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
- "value": 0
+ "value": 1
},
{
- "color": "#299c46",
- "value": 0
+ "color": "#bf1b00",
+ "value": 5
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
- "y": 5
+ "x": 0,
+ "y": 9
},
- "id": 18,
+ "id": 32,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
+ "hide": false,
"instant": true,
"interval": "",
- "intervalFactor": 1,
+ "intervalFactor": 2,
"legendFormat": "",
"refId": "A"
}
],
- "title": "Online Partitions",
+ "title": "Under Min ISR Partitions",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Unclean leader election rate",
+ "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#299c46",
+ "color": "#508642",
"value": null
},
{
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 2
+ "color": "#ef843c",
+ "value": 1
},
{
- "color": "#d44a3a"
+ "color": "#bf1b00",
+ "value": 1
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 12,
- "y": 5
+ "x": 0,
+ "y": 13
},
- "id": 16,
+ "id": 22,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"instant": true,
"interval": "",
"refId": "A"
}
],
- "title": "Unclean Leader Election Rate",
+ "title": "Offline Partitions Count",
"type": "stat"
},
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 31,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Throughput & Request rate",
+ "type": "row"
+ },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
"fieldConfig": {
"defaults": {
"color": {
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
- "axisLabel": "",
+ "axisLabel": "Bytes/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
- "spanNulls": true,
+ "spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
}
]
},
- "unit": "ops"
+ "unit": "Bps"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "kafka-3:80"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
- "h": 9,
+ "h": 8,
"w": 12,
- "x": 12,
- "y": 9
+ "x": 0,
+ "y": 18
},
- "id": 123,
+ "id": 84,
+ "links": [],
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
- "displayMode": "table",
+ "displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "editorMode": "code",
- "exemplar": false,
- "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Bytes in",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"hide": false,
"interval": "",
- "legendFormat": "{{instance}}",
- "range": true,
- "refId": "A"
+ "intervalFactor": 2,
+ "legendFormat": "Bytes out",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "B",
+ "step": 4
}
],
- "title": "IsrExpands per Sec",
+ "title": "Broker network throughput",
"type": "timeseries"
},
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 18
- },
- "id": 31,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Request rate",
- "type": "row"
- },
{
"datasource": {
"type": "prometheus",
"gridPos": {
"h": 4,
"w": 4,
- "x": 0,
- "y": 19
+ "x": 12,
+ "y": 18
},
"id": 93,
"links": [],
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Produce request rate.",
+ "description": "Fetch request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 4,
- "y": 19
+ "x": 16,
+ "y": 18
},
- "id": 35,
+ "id": 37,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Produce Request Per Sec",
+ "title": "Consumer Fetch Request Per Sec",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Fetch request rate.",
+ "description": "Produce request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
- "y": 19
+ "x": 20,
+ "y": 18
},
- "id": 37,
+ "id": 35,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Consumer Fetch Request Per Sec",
+ "title": "Produce Request Per Sec",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Fetch request rate.",
+ "description": "Metadata request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 0,
- "y": 23
+ "x": 12,
+ "y": 22
},
- "id": 94,
+ "id": 36,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Broker Fetch Request Per Sec",
+ "title": "Metadata Request Per Sec",
"type": "stat"
},
{
"gridPos": {
"h": 4,
"w": 4,
- "x": 4,
- "y": 23
+ "x": 16,
+ "y": 22
},
"id": 38,
"links": [],
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Metadata request rate.",
+ "description": "Fetch request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
- "y": 23
+ "x": 20,
+ "y": 22
},
- "id": 36,
+ "id": 94,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Metadata Request Per Sec",
+ "title": "Broker Fetch Request Per Sec",
"type": "stat"
},
{
"h": 1,
"w": 24,
"x": 0,
- "y": 27
+ "y": 26
},
"id": 40,
"panels": [],
]
},
"gridPos": {
- "h": 8,
+ "h": 12,
"w": 8,
"x": 0,
- "y": 28
+ "y": 27
},
"id": 27,
"links": [],
}
}
]
+ },
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
}
]
},
"gridPos": {
- "h": 8,
+ "h": 12,
"w": 8,
"x": 8,
- "y": 28
+ "y": 27
},
"id": 2,
"links": [],
]
},
"gridPos": {
- "h": 8,
+ "h": 12,
"w": 8,
"x": 16,
- "y": 28
+ "y": 27
},
"id": 3,
"links": [],
"h": 1,
"w": 24,
"x": 0,
- "y": 36
+ "y": 39
},
"id": 29,
"panels": [
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Messages In",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 24
- },
- "id": 5,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes In",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 24
- },
- "id": 6,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes Out",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Messages/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "cps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 32
- },
- "id": 10,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Messages In Per Broker",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 32
- },
- "id": 7,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes In Per Broker",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 32
- },
- "id": 9,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Bytes Out Per Broker",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Throughput In/Out",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 37
- },
- "id": 44,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 25
- },
- "id": 24,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
"interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
+ "intervalFactor": 2,
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
+ "refId": "A",
+ "step": 4
}
],
- "title": "Network Processor Avg Usage Percent",
+ "title": "Messages In",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "",
+ "axisLabel": "Bytes/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
- "spanNulls": true,
+ "spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
}
]
},
- "unit": "percentunit"
+ "unit": "Bps"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 25
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 24
},
- "id": 25,
+ "id": 5,
+ "links": [],
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "exemplar": true,
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
+ "intervalFactor": 2,
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
}
],
- "title": "Request Handler Avg Percent",
+ "title": "Bytes In",
"type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Thread utilization",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 38
- },
- "id": 86,
- "panels": [
+ },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Latency in millseconds for ZooKeeper requests from broker.\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "",
+ "axisLabel": "Bytes/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
- "spanNulls": true,
+ "spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"links": [],
"mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
}
]
},
- "unit": "ms"
+ "unit": "Bps"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 26
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 24
},
- "id": 88,
+ "id": 6,
+ "links": [],
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
+ "exemplar": true,
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
}
],
- "title": "Zookeeper Request Latency",
+ "title": "Bytes Out",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "",
+ "axisLabel": "Messages/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
- "spanNulls": true,
+ "spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
}
]
},
- "unit": "short"
+ "unit": "cps"
},
"overrides": []
},
"gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 26
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 32
},
- "id": 92,
+ "id": 10,
+ "links": [],
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "instant": false,
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"interval": "",
+ "intervalFactor": 2,
"legendFormat": "{{instance}}",
- "refId": "A"
+ "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
+ "refId": "A",
+ "step": 4
}
],
- "title": "Zookeeper connections per sec",
+ "title": "Messages In Per Broker",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
- "axisLabel": "",
+ "axisLabel": "Bytes/s",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"viz": false
},
"lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
- "spanNulls": true,
+ "spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
}
]
},
- "unit": "short"
+ "unit": "Bps"
},
"overrides": []
},
"gridPos": {
- "h": 9,
+ "h": 8,
"w": 8,
- "x": 0,
- "y": 35
+ "x": 8,
+ "y": 32
},
- "id": 89,
+ "id": 7,
+ "links": [],
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"interval": "",
+ "intervalFactor": 2,
"legendFormat": "{{instance}}",
- "refId": "A"
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
}
],
- "title": "Zookeeper expired connections per sec",
+ "title": "Bytes In Per Broker",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
"fieldConfig": {
"defaults": {
"color": {
}
]
},
- "unit": "short"
+ "unit": "binBps"
},
"overrides": []
},
"gridPos": {
- "h": 9,
+ "h": 8,
"w": 8,
- "x": 8,
- "y": 35
+ "x": 16,
+ "y": 32
},
- "id": 90,
+ "id": 9,
+ "links": [],
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"interval": "",
+ "intervalFactor": 1,
"legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "Zookeeper disconnect per sec",
- "type": "timeseries"
+ "title": "Bytes Out Per Broker",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Throughput In/Out",
+ "type": "row"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 44,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Thread utilization",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 12,
+ "x": 0,
+ "y": 41
+ },
+ "id": 24,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 16,
- "y": 35
+ "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Network Processor Avg Usage Percent",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 91,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 12,
+ "x": 12,
+ "y": 41
+ },
+ "id": 25,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Zookeeper auth failures per sec",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "Zookeeper",
- "type": "row"
+ "title": "Request Handler Avg Percent",
+ "type": "timeseries"
},
{
"collapsed": false,
"h": 1,
"w": 24,
"x": 0,
- "y": 39
+ "y": 53
},
"id": 82,
"panels": [],
},
"unit": "ops"
},
- "overrides": []
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 40
+ "y": 54
},
"id": 80,
"options": {
},
"unit": "ops"
},
- "overrides": []
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 40
+ "y": 54
},
"id": 83,
"options": {
"h": 1,
"w": 24,
"x": 0,
- "y": 49
+ "y": 63
},
"id": 53,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 50
+ "y": 64
},
"id": 58,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 51
+ "y": 65
},
"id": 68,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 52
+ "y": 66
},
"id": 66,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 53
+ "y": 67
},
"id": 102,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 54
+ "y": 68
},
"id": 120,
"panels": [
"h": 1,
"w": 24,
"x": 0,
- "y": 55
+ "y": 69
},
"id": 46,
"panels": [
]
},
"time": {
- "from": "now-12h",
+ "from": "now-5m",
"to": "now"
},
"timepicker": {
projects / demos / kafka / monitoring / blobdiff