"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of active controllers in the cluster.",
+ "description": "Number of Brokers Online",
"fieldConfig": {
"defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
"mappings": [
{
"options": {
"mode": "absolute",
"steps": [
{
- "color": "#299c46",
+ "color": "#d44a3a",
"value": null
},
{
- "color": "#e5ac0e",
- "value": 2
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
},
{
- "color": "#bf1b00"
+ "color": "#299c46",
+ "value": 2
}
]
},
"x": 0,
"y": 1
},
- "id": 12,
+ "id": 14,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
- "orientation": "vertical",
+ "orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
"values": false
},
"text": {},
- "textMode": "value_and_name"
+ "textMode": "auto"
},
"pluginVersion": "10.0.3",
+ "repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
+ "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 1,
- "legendFormat": "{{instance}}",
+ "legendFormat": "",
"refId": "A"
}
],
- "title": "Active Controllers",
+ "title": "Brokers Online",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of partitions under min insync replicas.",
+ "description": "Number of active controllers in the cluster.",
"fieldConfig": {
"defaults": {
- "color": {
- "mode": "thresholds"
- },
"mappings": [
{
"options": {
"mode": "absolute",
"steps": [
{
- "color": "#508642",
+ "color": "#299c46",
"value": null
},
{
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 1
+ "color": "#e5ac0e",
+ "value": 2
},
{
- "color": "#bf1b00",
- "value": 5
+ "color": "#bf1b00"
}
]
},
},
"gridPos": {
"h": 4,
- "w": 4,
+ "w": 5,
"x": 4,
"y": 1
},
- "id": 32,
+ "id": 12,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
- "orientation": "horizontal",
+ "orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
"values": false
},
"text": {},
- "textMode": "auto"
+ "textMode": "value_and_name"
},
"pluginVersion": "10.0.3",
"targets": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
"format": "time_series",
- "hide": false,
"instant": true,
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "Under Min ISR Partitions",
+ "title": "Active Controllers",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
+ "description": "Partitions that are online",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#508642",
+ "color": "#d44a3a",
"value": null
},
{
- "color": "#ef843c",
- "value": 1
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 0
},
{
- "color": "#bf1b00",
- "value": 1
+ "color": "#299c46",
+ "value": 0
}
]
},
},
"gridPos": {
"h": 4,
- "w": 4,
- "x": 8,
+ "w": 6,
+ "x": 9,
"y": 1
},
- "id": 22,
+ "id": 18,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"instant": true,
"interval": "",
"refId": "A"
}
],
- "title": "Offline Partitions Count",
+ "title": "Online Partitions",
"type": "stat"
},
{
"gridPos": {
"h": 4,
"w": 4,
- "x": 12,
+ "x": 15,
"y": 1
},
"id": 33,
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "description": "Unclean leader election rate",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "palette-classic"
+ "mode": "thresholds"
},
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
}
- },
- "links": [],
- "mappings": [],
+ ],
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "green",
+ "color": "#299c46",
"value": null
},
{
- "color": "red",
- "value": 80
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 2
+ },
+ {
+ "color": "#d44a3a"
}
]
},
- "unit": "Bps"
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
+ "h": 4,
+ "w": 4,
+ "x": 19,
"y": 1
},
- "id": 84,
+ "id": 16,
"links": [],
+ "maxDataPoints": 100,
"options": {
- "legend": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
"calcs": [
- "mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "fields": "",
+ "values": false
},
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
+ "text": {},
+ "textMode": "auto"
},
- "pluginVersion": "8.1.3",
+ "pluginVersion": "10.0.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "Bytes in",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
- "hide": false,
+ "instant": true,
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "Bytes out",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "B",
- "step": 4
+ "intervalFactor": 1,
+ "legendFormat": "",
+ "refId": "A"
}
],
- "title": "Broker network throughput",
- "type": "timeseries"
+ "title": "Unclean Leader Election Rate",
+ "type": "stat"
},
{
"datasource": {
"maxDataPoints": 100,
"options": {
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "editorMode": "code",
+ "exemplar": false,
"expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
"hide": false,
- "instant": true,
+ "instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "",
+ "range": true,
"refId": "A"
}
],
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of Brokers Online",
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
+ "decimals": 0,
+ "links": [],
"mappings": [
{
"options": {
- "match": "null",
- "result": {
+ "null": {
+ "index": 0,
"text": "N/A"
}
},
- "type": "special"
+ "type": "value"
}
],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "#d44a3a",
+ "color": "green",
"value": null
},
{
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 0
- },
- {
- "color": "#299c46",
- "value": 2
+ "color": "red",
+ "value": 80
}
]
},
- "unit": "none"
+ "unit": "partitions"
},
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 4,
+ "h": 12,
+ "w": 11,
"x": 4,
"y": 5
},
- "id": 14,
- "links": [],
- "maxDataPoints": 100,
+ "id": 123,
"options": {
- "colorMode": "value",
- "graphMode": "none",
- "justifyMode": "auto",
- "orientation": "horizontal",
+ "legend": {
+ "calcs": [
+ "last",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Underreplicated Partitions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "decimals": 0,
+ "links": [],
+ "mappings": [
+ {
+ "options": {
+ "null": {
+ "index": 0,
+ "text": "0"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "min": 0,
+ "unit": "partitions"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 15,
+ "y": 5
+ },
+ "id": 124,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "value"
+ ]
+ },
+ "pieType": "pie",
"reduceOptions": {
"calcs": [
- "lastNotNull"
+ "last"
],
"fields": "",
"values": false
},
- "text": {},
- "textMode": "auto"
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "pluginVersion": "10.0.3",
- "repeatDirection": "h",
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
- "format": "time_series",
- "instant": true,
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
+ "hide": false,
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "",
+ "legendFormat": "{{instance}}",
+ "range": true,
"refId": "A"
}
],
- "title": "Brokers Online",
- "type": "stat"
+ "title": "Leader Count",
+ "type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Partitions that are online",
+ "description": "Number of partitions under min insync replicas.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#d44a3a",
+ "color": "#508642",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
- "value": 0
+ "value": 1
},
{
- "color": "#299c46",
- "value": 0
+ "color": "#bf1b00",
+ "value": 5
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
- "y": 5
+ "x": 0,
+ "y": 9
},
- "id": 18,
+ "id": 32,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "editorMode": "code",
+ "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
- "instant": true,
+ "hide": false,
+ "instant": false,
"interval": "",
- "intervalFactor": 1,
+ "intervalFactor": 2,
"legendFormat": "",
+ "range": true,
"refId": "A"
}
],
- "title": "Online Partitions",
+ "title": "Under Min ISR Partitions",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Unclean leader election rate",
+ "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "absolute",
"steps": [
{
- "color": "#299c46",
+ "color": "#508642",
"value": null
},
{
- "color": "rgba(237, 129, 40, 0.89)",
- "value": 2
+ "color": "#ef843c",
+ "value": 1
},
{
- "color": "#d44a3a"
+ "color": "#bf1b00",
+ "value": 1
}
]
},
"gridPos": {
"h": 4,
"w": 4,
- "x": 12,
- "y": 5
+ "x": 0,
+ "y": 13
},
- "id": 16,
+ "id": 22,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
- "graphMode": "none",
+ "graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "editorMode": "code",
+ "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
"format": "time_series",
- "instant": true,
+ "instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
+ "range": true,
"refId": "A"
}
],
- "title": "Unclean Leader Election Rate",
+ "title": "Offline Partitions Count",
"type": "stat"
},
{
+ "collapsed": true,
"datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "type": "datasource",
+ "uid": "grafana"
},
- "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 40,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "kafka-3:80"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Cores",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
"legend": false,
"tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 9
- },
- "id": 123,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "IsrExpands per Sec",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 18
- },
- "id": 31,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Request rate",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Produce request rate.",
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": [
{
- "color": "green",
- "value": null
+ "matcher": {
+ "id": "byName",
+ "options": "localhost:7071"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#629E51",
+ "mode": "fixed"
+ }
+ }
+ ]
}
]
},
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 0,
- "y": 19
- },
- "id": 93,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "10.0.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 0,
+ "y": 19
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "All Request Per Sec",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Produce request rate.",
- "fieldConfig": {
- "defaults": {
- "mappings": [
+ "id": 27,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "type": "special"
+ "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "metric": "process_cpu_secondspersec",
+ "refId": "A",
+ "step": 4
}
],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 4,
- "y": 19
- },
- "id": 35,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
+ "title": "CPU Usage",
+ "type": "timeseries"
},
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "10.0.3",
- "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Produce Request Per Sec",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Fetch request rate.",
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Memory",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [
{
- "color": "green",
- "value": null
+ "matcher": {
+ "id": "byName",
+ "options": "localhost:7071"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#BA43A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
}
]
},
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 8,
- "y": 19
- },
- "id": 37,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 8,
+ "y": 19
+ },
+ "id": 2,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "metric": "jvm_memory_bytes_used",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "B"
+ }
],
- "fields": "",
- "values": false
+ "title": "JVM Memory Used",
+ "type": "timeseries"
},
- "text": {},
- "textMode": "auto"
- },
- "pluginVersion": "10.0.3",
- "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
- "interval": "",
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Consumer Fetch Request Per Sec",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Fetch request rate.",
- "fieldConfig": {
- "defaults": {
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "% time in GC",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [
{
- "color": "green",
- "value": null
+ "matcher": {
+ "id": "byName",
+ "options": "localhost:7071"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#890F02",
+ "mode": "fixed"
+ }
+ }
+ ]
}
]
},
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 0,
- "y": 23
- },
- "id": 94,
- "links": [],
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 16,
+ "y": 19
+ },
+ "id": 3,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "metric": "jvm_gc_collection_seconds_sum",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "title": "Time spent in GC",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "System",
+ "type": "row"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 18
+ },
+ "id": 31,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Throughput & Request rate",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Bytes/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
},
- "text": {},
- "textMode": "auto"
+ "overrides": []
},
- "pluginVersion": "10.0.3",
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "id": 84,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
+ "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
"interval": "",
- "legendFormat": "",
- "refId": "A"
+ "intervalFactor": 2,
+ "legendFormat": "Bytes in",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Bytes out",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "B",
+ "step": 4
}
],
- "title": "Broker Fetch Request Per Sec",
- "type": "stat"
+ "title": "Broker network throughput",
+ "type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Offset Commit request rate.",
+ "description": "Produce request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 4,
- "y": 23
+ "x": 12,
+ "y": 19
},
- "id": 38,
+ "id": 93,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Offset Commit Request Per Sec",
+ "title": "All Request Per Sec",
"type": "stat"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Metadata request rate.",
+ "description": "Fetch request rate.",
"fieldConfig": {
"defaults": {
"mappings": [
"gridPos": {
"h": 4,
"w": 4,
- "x": 8,
- "y": 23
+ "x": 16,
+ "y": 19
},
- "id": 36,
+ "id": 37,
"links": [],
"maxDataPoints": 100,
"options": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
- "title": "Metadata Request Per Sec",
+ "title": "Consumer Fetch Request Per Sec",
"type": "stat"
},
- {
- "collapsed": false,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 27
- },
- "id": 40,
- "panels": [],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "System",
- "type": "row"
- },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "description": "Produce request rate.",
"fieldConfig": {
"defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "Cores",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
}
- },
- "links": [],
- "mappings": [],
+ ],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
- },
- {
- "color": "red",
- "value": 80
}
]
},
- "unit": "percent"
+ "unit": "none"
},
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "localhost:7071"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#629E51",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 28
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 19
},
- "id": 27,
+ "id": 35,
"links": [],
+ "maxDataPoints": 100,
"options": {
- "legend": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
"calcs": [
- "mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "fields": "",
+ "values": false
},
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
+ "text": {},
+ "textMode": "auto"
},
- "pluginVersion": "8.1.3",
+ "pluginVersion": "10.0.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
- "format": "time_series",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "process_cpu_secondspersec",
- "refId": "A",
- "step": 4
+ "legendFormat": "",
+ "refId": "A"
}
],
- "title": "CPU Usage",
- "type": "timeseries"
+ "title": "Produce Request Per Sec",
+ "type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "description": "Metadata request rate.",
"fieldConfig": {
"defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "Memory",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
}
- },
- "links": [],
- "mappings": [],
+ ],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
- },
- {
- "color": "red",
- "value": 80
}
]
},
- "unit": "bytes"
+ "unit": "none"
},
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "localhost:7071"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#BA43A9",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 28
+ "h": 4,
+ "w": 4,
+ "x": 12,
+ "y": 23
},
- "id": 2,
+ "id": 36,
"links": [],
+ "maxDataPoints": 100,
"options": {
- "legend": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
"calcs": [
- "mean",
- "lastNotNull",
- "max"
+ "lastNotNull"
],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "fields": "",
+ "values": false
},
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
+ "text": {},
+ "textMode": "auto"
},
- "pluginVersion": "8.1.3",
+ "pluginVersion": "10.0.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "jvm_memory_bytes_used",
- "refId": "A",
- "step": 4
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
"interval": "",
- "legendFormat": "{{instance}}",
- "refId": "B"
+ "legendFormat": "",
+ "refId": "A"
}
],
- "title": "JVM Memory Used",
- "type": "timeseries"
+ "title": "Metadata Request Per Sec",
+ "type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
+ "description": "Offset Commit request rate.",
"fieldConfig": {
"defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "% time in GC",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
}
- },
- "links": [],
- "mappings": [],
+ ],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
- },
- {
- "color": "red",
- "value": 80
}
]
},
- "unit": "percentunit"
+ "unit": "none"
},
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "localhost:7071"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "#890F02",
- "mode": "fixed"
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 16,
+ "y": 23
+ },
+ "id": 38,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Offset Commit Request Per Sec",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Fetch request rate.",
+ "fieldConfig": {
+ "defaults": {
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
}
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
}
]
- }
- ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 23
+ },
+ "id": 94,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "text": {},
+ "textMode": "auto"
+ },
+ "pluginVersion": "10.0.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "A"
+ }
+ ],
+ "title": "Broker Fetch Request Per Sec",
+ "type": "stat"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 29,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Throughput In/Out",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Messages/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "cps"
+ },
+ "overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
- "x": 16,
+ "x": 0,
"y": 28
},
- "id": 3,
+ "id": 4,
"links": [],
"options": {
"legend": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
+ "exemplar": true,
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
"interval": "",
"intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "jvm_gc_collection_seconds_sum",
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
"refId": "A",
"step": 4
}
],
- "title": "Time spent in GC",
+ "title": "Messages In",
"type": "timeseries"
},
{
- "collapsed": true,
"datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 36
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "id": 29,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Messages/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "cps"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Bytes/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 24
- },
- "id": 4,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "tooltip": {
- "mode": "single"
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
- "refId": "A",
- "step": 4
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 28
+ },
+ "id": 5,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Messages In",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 24
+ "exemplar": true,
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "title": "Bytes In",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 5,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Bytes/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes In",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 24
- },
- "id": 6,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "exemplar": true,
- "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "bytes/sec",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes Out",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Messages/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "cps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 32
- },
- "id": 10,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Messages In Per Broker",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "Bytes/s",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "Bps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 32
- },
- "id": 7,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{instance}}",
- "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
- "refId": "A",
- "step": 4
- }
- ],
- "title": "Bytes In Per Broker",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 32
- },
- "id": 9,
- "links": [],
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Bytes Out Per Broker",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Throughput In/Out",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 37
- },
- "id": 44,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 25
- },
- "id": 24,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Network Processor Avg Usage Percent",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 25
- },
- "id": 25,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "title": "Request Handler Avg Percent",
- "type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Thread utilization",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "unit": "Bps"
+ },
+ "overrides": []
},
"gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 38
- },
- "id": 86,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "Latency in millseconds for ZooKeeper requests from broker.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 26
- },
- "id": 88,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
- ],
- "title": "Zookeeper Request Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 26
- },
- "id": 92,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "instant": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 28
+ },
+ "id": 6,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Zookeeper connections per sec",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 0,
- "y": 35
+ "exemplar": true,
+ "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "bytes/sec",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "title": "Bytes Out",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 89,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Messages/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "cps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 36
+ },
+ "id": 10,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Zookeeper expired connections per sec",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 8,
- "y": 35
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "title": "Messages In Per Broker",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 90,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Bytes/s",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 36
+ },
+ "id": 7,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Zookeeper disconnect per sec",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 16,
- "y": 35
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{instance}}",
+ "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "title": "Bytes In Per Broker",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 91,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 36
+ },
+ "id": 9,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Zookeeper auth failures per sec",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "Zookeeper",
- "type": "row"
+ "title": "Bytes Out Per Broker",
+ "type": "timeseries"
},
{
"collapsed": false,
"h": 1,
"w": 24,
"x": 0,
- "y": 39
+ "y": 44
},
- "id": 82,
+ "id": 44,
"panels": [],
"targets": [
{
"refId": "A"
}
],
- "title": "Isr Shrinks / Expands",
+ "title": "Thread utilization",
"type": "row"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
+ "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
"fieldConfig": {
"defaults": {
"color": {
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
}
]
},
- "unit": "ops"
+ "unit": "percentunit"
},
"overrides": []
},
"gridPos": {
- "h": 9,
+ "h": 12,
"w": 12,
"x": 0,
- "y": 40
+ "y": 45
},
- "id": 80,
+ "id": 24,
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
+ "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "IsrShrinks per Sec",
+ "title": "Network Processor Avg Usage Percent",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
+ "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
"fieldConfig": {
"defaults": {
"color": {
}
]
},
- "unit": "ops"
+ "unit": "percentunit"
},
"overrides": []
},
"gridPos": {
- "h": 9,
+ "h": 12,
"w": 12,
"x": 12,
- "y": 40
+ "y": 45
},
- "id": 83,
+ "id": 25,
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
- "hide": false,
+ "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "IsrExpands per Sec",
+ "title": "Request Handler Avg Percent",
"type": "timeseries"
},
{
- "collapsed": true,
+ "collapsed": false,
"datasource": {
"type": "datasource",
"uid": "grafana"
"h": 1,
"w": 24,
"x": 0,
- "y": 49
+ "y": 57
},
- "id": 53,
- "panels": [
+ "id": 82,
+ "panels": [],
+ "targets": [
{
"datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
+ "type": "datasource",
+ "uid": "grafana"
},
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 28
+ "refId": "A"
+ }
+ ],
+ "title": "Isr Shrinks / Expands",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 55,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
- "legendFormat": "{{topic}}",
- "refId": "A"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
- "title": "Log size per Topic",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 28
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
- "id": 56,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
},
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 58
+ },
+ "id": 80,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Log size per Broker",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
+ "interval": "",
+ "legendFormat": "{{instance}}",
"refId": "A"
}
],
- "title": "Logs size",
- "type": "row"
+ "title": "IsrShrinks per Sec",
+ "type": "timeseries"
},
{
- "collapsed": true,
"datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 50
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "id": 58,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 29
- },
- "id": 60,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
- ],
- "title": "Producer - RequestQueueTimeMs",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 29
+ "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 61,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "kafka-2:80"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 58
+ },
+ "id": 83,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Producer - LocalTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
+ "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "IsrExpands per Sec",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 67
+ },
+ "id": 53,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
},
- "gridPos": {
- "h": 10,
- "w": 8,
- "x": 0,
- "y": 38
+ "refId": "A"
+ }
+ ],
+ "title": "Logs size",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 62,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 68
+ },
+ "id": 55,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Producer - RemoteTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 8,
- "x": 8,
- "y": 38
+ "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
+ "legendFormat": "{{topic}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Log size per Topic",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 63,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 68
+ },
+ "id": 56,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Producer - ResponseQueueTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 8,
- "x": 16,
- "y": 38
- },
- "id": 64,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
- },
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
- ],
- "title": "Producer - ResponseSendTimeMs",
- "type": "timeseries"
+ "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
}
],
+ "title": "Log size per Broker",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 77
+ },
+ "id": 58,
+ "panels": [],
"targets": [
{
"datasource": {
"type": "row"
},
{
- "collapsed": true,
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
},
"gridPos": {
- "h": 1,
- "w": 24,
+ "h": 9,
+ "w": 12,
"x": 0,
- "y": 51
+ "y": 78
},
- "id": 68,
- "panels": [
+ "id": 60,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 30
+ "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Producer - RequestQueueTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 69,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 78
+ },
+ "id": 61,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Consumer - RequestQueueTimeMs",
- "type": "timeseries"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 12,
- "y": 30
+ "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Producer - LocalTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 70,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 0,
+ "y": 87
+ },
+ "id": 62,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Consumer - LocalTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
+ "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Producer - RemoteTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 0,
- "y": 39
- },
- "id": 71,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "tooltip": {
- "mode": "single"
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 8,
+ "y": 87
+ },
+ "id": 63,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Consumer - RemoteTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 8,
- "y": 39
+ "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Producer - ResponseQueueTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 72,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 16,
+ "y": 87
+ },
+ "id": 64,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Consumer - ResponseQueueTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Producer - ResponseSendTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 97
+ },
+ "id": 68,
+ "panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
+ "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
"fieldConfig": {
"defaults": {
"color": {
},
"gridPos": {
"h": 9,
- "w": 8,
- "x": 16,
- "y": 39
+ "w": 12,
+ "x": 0,
+ "y": 30
},
- "id": 73,
+ "id": 69,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
- "displayMode": "table",
+ "displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
+ "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
"hide": false,
"legendFormat": "{{instance}} - {{quantile}}",
"refId": "A"
}
],
- "title": "Consumer - ResponseSendTimeMs",
+ "title": "Consumer - RequestQueueTimeMs",
"type": "timeseries"
- }
- ],
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "refId": "A"
- }
- ],
- "title": "Consumer Performance",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 52
- },
- "id": 66,
- "panels": [
+ },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
+ "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
"fieldConfig": {
"defaults": {
"color": {
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
"gridPos": {
"h": 9,
"w": 12,
- "x": 0,
- "y": 31
+ "x": 12,
+ "y": 30
},
- "id": 74,
+ "id": 70,
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
"hide": false,
"legendFormat": "{{instance}} - {{quantile}}",
"refId": "A"
}
],
- "title": "FetchFollower - RequestQueueTimeMs",
+ "title": "Consumer - LocalTimeMs",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
+ "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
"fieldConfig": {
"defaults": {
"color": {
},
"links": [],
"mappings": [],
- "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
},
"gridPos": {
"h": 9,
- "w": 12,
- "x": 12,
- "y": 31
+ "w": 8,
+ "x": 0,
+ "y": 39
},
- "id": 75,
+ "id": 71,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
- "displayMode": "list",
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
"hide": false,
"legendFormat": "{{instance}} - {{quantile}}",
"refId": "A"
}
],
- "title": "FetchFollower - LocalTimeMs",
+ "title": "Consumer - RemoteTimeMs",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
+ "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
"fieldConfig": {
"defaults": {
"color": {
},
"links": [],
"mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
"gridPos": {
"h": 9,
"w": 8,
- "x": 0,
- "y": 40
+ "x": 8,
+ "y": 39
},
- "id": 76,
+ "id": 72,
"options": {
"legend": {
"calcs": [
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
"hide": false,
"legendFormat": "{{instance}} - {{quantile}}",
"refId": "A"
}
],
- "title": "FetchFollower - RemoteTimeMs",
+ "title": "Consumer - ResponseQueueTimeMs",
"type": "timeseries"
},
{
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
+ "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
"fieldConfig": {
"defaults": {
"color": {
},
"links": [],
"mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
"gridPos": {
"h": 9,
"w": 8,
- "x": 8,
- "y": 40
+ "x": 16,
+ "y": 39
+ },
+ "id": 73,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Consumer - ResponseSendTimeMs",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Consumer Performance",
+ "type": "row"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 98
+ },
+ "id": 66,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Fetch Follower Performance",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 99
+ },
+ "id": 74,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "id": 77,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "FetchFollower - RequestQueueTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 99
+ },
+ "id": 75,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "FetchFollower - ResponseQueueTimeMs",
- "type": "timeseries"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 9,
- "w": 8,
- "x": 16,
- "y": 40
+ "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "FetchFollower - LocalTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 78,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
- "hide": false,
- "legendFormat": "{{instance}} - {{quantile}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 8,
+ "x": 0,
+ "y": 108
+ },
+ "id": 76,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "FetchFollower - ResponseSendTimeMs",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
"refId": "A"
}
],
- "title": "Fetch Follower Performance",
- "type": "row"
+ "title": "FetchFollower - RemoteTimeMs",
+ "type": "timeseries"
},
{
- "collapsed": true,
"datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 53
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "id": 102,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 32
- },
- "id": 98,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "tooltip": {
- "mode": "single"
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
- "interval": "",
- "legendFormat": "{{listener}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 8,
+ "x": 8,
+ "y": 108
+ },
+ "id": 77,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections count per listener",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
+ "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "FetchFollower - ResponseQueueTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 32
- },
- "id": 100,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
},
- "tooltip": {
- "mode": "single"
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 8,
+ "x": 16,
+ "y": 108
+ },
+ "id": 78,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections count per broker",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
+ "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} - {{quantile}}",
+ "refId": "A"
+ }
+ ],
+ "title": "FetchFollower - ResponseSendTimeMs",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 117
+ },
+ "id": 102,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 40
+ "refId": "A"
+ }
+ ],
+ "title": "Connections",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 104,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
- "interval": "",
- "legendFormat": "{{listener}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 118
+ },
+ "id": 98,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections creation rate per listener",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 40
+ "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
+ "interval": "",
+ "legendFormat": "{{listener}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections count per listener",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 106,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 118
+ },
+ "id": 100,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections creation rate per instance",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 48
+ "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections count per broker",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 108,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
},
- "tooltip": {
- "mode": "single"
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
- "interval": "",
- "legendFormat": "{{listener}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 126
+ },
+ "id": 104,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections close rate per listener",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 48
+ "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
+ "interval": "",
+ "legendFormat": "{{listener}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections creation rate per listener",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 110,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 126
+ },
+ "id": 106,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections creation rate per instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 134
+ },
+ "id": 108,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections close rate per instance",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percent"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 56
+ "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
+ "interval": "",
+ "legendFormat": "{{listener}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections close rate per listener",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 112,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
- "interval": "",
- "legendFormat": "{{instance}} - {{listener}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 134
+ },
+ "id": 110,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Acceptor Blocked Percentage",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
+ "uid": "PBFA97CFB590B2093"
},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 56
+ "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections close rate per instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 114,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
- "interval": "",
- "legendFormat": "{{client_software_name}} {{client_software_version}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 142
+ },
+ "id": 112,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Connections per client version",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{instance}} - {{listener}}",
"refId": "A"
}
],
- "title": "Connections",
- "type": "row"
+ "title": "Acceptor Blocked Percentage",
+ "type": "timeseries"
},
{
- "collapsed": true,
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
},
"gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 54
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 142
},
- "id": 120,
- "panels": [
+ "id": 114,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of consumer groups per group coordinator",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
+ "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
+ "interval": "",
+ "legendFormat": "{{client_software_name}} {{client_software_version}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Connections per client version",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 150
+ },
+ "id": 120,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 33
+ "refId": "A"
+ }
+ ],
+ "title": "Group Coordinator",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Number of consumer groups per group coordinator",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 116,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
- "instant": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 151
+ },
+ "id": 116,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Consumer groups number per coordinator",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "Number of consumer group per state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 33
- },
- "id": 118,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single"
- }
+ "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Consumer groups number per coordinator",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Number of consumer group per state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
- "instant": false,
- "interval": "",
- "legendFormat": "stable",
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "preparing-rebalance",
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "dead",
- "refId": "C"
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "completing-rebalance",
- "refId": "D"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
- "interval": "",
- "legendFormat": "empty",
- "refId": "E"
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 151
+ },
+ "id": 118,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Nb consumer groups per state",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
+ "pluginVersion": "8.1.3",
"targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
+ "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "stable",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "preparing-rebalance",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "dead",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "completing-rebalance",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
+ "interval": "",
+ "legendFormat": "empty",
+ "refId": "E"
}
],
- "title": "Group Coordinator",
- "type": "row"
+ "title": "Nb consumer groups per state",
+ "type": "timeseries"
},
{
- "collapsed": true,
+ "collapsed": false,
"datasource": {
"type": "datasource",
"uid": "grafana"
"h": 1,
"w": 24,
"x": 0,
- "y": 55
+ "y": 159
},
"id": 46,
- "panels": [
+ "panels": [],
+ "targets": [
{
"datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
- },
- "description": "The number of messages produced converted to match the log.message.format.version.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
+ "type": "datasource",
+ "uid": "grafana"
},
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 0,
- "y": 34
+ "refId": "A"
+ }
+ ],
+ "title": "Message Conversion",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "The number of messages produced converted to match the log.message.format.version.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 48,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
- "hide": false,
- "interval": "",
- "legendFormat": "{{topic}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 160
+ },
+ "id": 48,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Number of produced message conversion",
- "type": "timeseries"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
- "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "links": [],
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 8,
- "y": 34
+ "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{topic}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of produced message conversion",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 51,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max"
- ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "mode": "single"
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "8.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PBFA97CFB590B2093"
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
- "hide": false,
- "interval": "",
- "legendFormat": "{{topic}}",
- "refId": "A"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 160
+ },
+ "id": 51,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max"
],
- "title": "Number of consumed message conversion",
- "type": "timeseries"
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.3",
+ "targets": [
{
"datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "description": "Number of connection per client version",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "decimals": 0,
- "mappings": [],
- "unit": "short"
- },
- "overrides": []
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
},
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 34
+ "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{topic}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of consumed message conversion",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "description": "Number of connection per client version",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "id": 96,
- "links": [],
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "values": [
- "value"
- ]
- },
- "pieType": "pie",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "tooltip": {
- "mode": "single"
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
}
},
- "targets": [
- {
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
- "interval": "",
- "legendFormat": "{{client_software_name}} - {{client_software_version}}",
- "refId": "A"
- }
+ "decimals": 0,
+ "mappings": [],
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 160
+ },
+ "id": 96,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "values": [
+ "value"
+ ]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
],
- "title": "Client version repartition",
- "type": "piechart"
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
+ },
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
+ "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
+ "interval": "",
+ "legendFormat": "{{client_software_name}} - {{client_software_version}}",
"refId": "A"
}
],
- "title": "Message Conversion",
- "type": "row"
+ "title": "Client version repartition",
+ "type": "piechart"
}
],
"refresh": "5s",
]
},
"time": {
- "from": "now-12h",
+ "from": "now-5m",
"to": "now"
},
"timepicker": {
projects / demos / kafka / monitoring / blobdiff