Strimzi - https://strimzi.io/quickstarts/ (with only 1Gi) -- WORKS
[demos/kafka/monitoring] / kafka-brokers.json
1 {
2   "annotations": {
3     "list": [
4       {
5         "builtIn": 1,
6         "datasource": {
7           "type": "datasource",
8           "uid": "grafana"
9         },
10         "enable": true,
11         "hide": true,
12         "iconColor": "rgba(0, 211, 255, 1)",
13         "name": "Annotations & Alerts",
14         "target": {
15           "limit": 100,
16           "matchAny": false,
17           "tags": [],
18           "type": "dashboard"
19         },
20         "type": "dashboard"
21       }
22     ]
23   },
24   "description": "Kafka resource usage and throughput",
25   "editable": true,
26   "fiscalYearStartMonth": 0,
27   "gnetId": 721,
28   "graphTooltip": 0,
29   "links": [],
30   "liveNow": false,
31   "panels": [
32     {
33       "collapsed": false,
34       "datasource": {
35         "type": "datasource",
36         "uid": "grafana"
37       },
38       "gridPos": {
39         "h": 1,
40         "w": 24,
41         "x": 0,
42         "y": 0
43       },
44       "id": 42,
45       "panels": [],
46       "targets": [
47         {
48           "datasource": {
49             "type": "datasource",
50             "uid": "grafana"
51           },
52           "refId": "A"
53         }
54       ],
55       "title": "Healthcheck",
56       "type": "row"
57     },
58     {
59       "datasource": {
60         "type": "prometheus",
61         "uid": "PBFA97CFB590B2093"
62       },
63       "description": "Number of Brokers Online",
64       "fieldConfig": {
65         "defaults": {
66           "color": {
67             "mode": "thresholds"
68           },
69           "mappings": [
70             {
71               "options": {
72                 "match": "null",
73                 "result": {
74                   "text": "N/A"
75                 }
76               },
77               "type": "special"
78             }
79           ],
80           "thresholds": {
81             "mode": "absolute",
82             "steps": [
83               {
84                 "color": "#d44a3a",
85                 "value": null
86               },
87               {
88                 "color": "rgba(237, 129, 40, 0.89)",
89                 "value": 0
90               },
91               {
92                 "color": "#299c46",
93                 "value": 2
94               }
95             ]
96           },
97           "unit": "none"
98         },
99         "overrides": []
100       },
101       "gridPos": {
102         "h": 4,
103         "w": 4,
104         "x": 0,
105         "y": 1
106       },
107       "id": 14,
108       "links": [],
109       "maxDataPoints": 100,
110       "options": {
111         "colorMode": "value",
112         "graphMode": "none",
113         "justifyMode": "auto",
114         "orientation": "horizontal",
115         "reduceOptions": {
116           "calcs": [
117             "lastNotNull"
118           ],
119           "fields": "",
120           "values": false
121         },
122         "text": {},
123         "textMode": "auto"
124       },
125       "pluginVersion": "10.0.3",
126       "repeatDirection": "h",
127       "targets": [
128         {
129           "datasource": {
130             "type": "prometheus",
131             "uid": "PBFA97CFB590B2093"
132           },
133           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
134           "format": "time_series",
135           "instant": true,
136           "interval": "",
137           "intervalFactor": 1,
138           "legendFormat": "",
139           "refId": "A"
140         }
141       ],
142       "title": "Brokers Online",
143       "type": "stat"
144     },
145     {
146       "datasource": {
147         "type": "prometheus",
148         "uid": "PBFA97CFB590B2093"
149       },
150       "description": "Number of active controllers in the cluster.",
151       "fieldConfig": {
152         "defaults": {
153           "mappings": [
154             {
155               "options": {
156                 "match": "null",
157                 "result": {
158                   "text": "N/A"
159                 }
160               },
161               "type": "special"
162             }
163           ],
164           "thresholds": {
165             "mode": "absolute",
166             "steps": [
167               {
168                 "color": "#299c46",
169                 "value": null
170               },
171               {
172                 "color": "#e5ac0e",
173                 "value": 2
174               },
175               {
176                 "color": "#bf1b00"
177               }
178             ]
179           },
180           "unit": "none"
181         },
182         "overrides": []
183       },
184       "gridPos": {
185         "h": 4,
186         "w": 5,
187         "x": 4,
188         "y": 1
189       },
190       "id": 12,
191       "links": [],
192       "maxDataPoints": 100,
193       "options": {
194         "colorMode": "value",
195         "graphMode": "none",
196         "justifyMode": "auto",
197         "orientation": "vertical",
198         "reduceOptions": {
199           "calcs": [
200             "lastNotNull"
201           ],
202           "fields": "",
203           "values": false
204         },
205         "text": {},
206         "textMode": "value_and_name"
207       },
208       "pluginVersion": "10.0.3",
209       "targets": [
210         {
211           "datasource": {
212             "type": "prometheus",
213             "uid": "PBFA97CFB590B2093"
214           },
215           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
216           "format": "time_series",
217           "instant": true,
218           "interval": "",
219           "intervalFactor": 1,
220           "legendFormat": "{{instance}}",
221           "refId": "A"
222         }
223       ],
224       "title": "Active Controllers",
225       "type": "stat"
226     },
227     {
228       "datasource": {
229         "type": "prometheus",
230         "uid": "PBFA97CFB590B2093"
231       },
232       "description": "Partitions that are online",
233       "fieldConfig": {
234         "defaults": {
235           "color": {
236             "mode": "thresholds"
237           },
238           "mappings": [
239             {
240               "options": {
241                 "match": "null",
242                 "result": {
243                   "text": "N/A"
244                 }
245               },
246               "type": "special"
247             }
248           ],
249           "thresholds": {
250             "mode": "absolute",
251             "steps": [
252               {
253                 "color": "#d44a3a",
254                 "value": null
255               },
256               {
257                 "color": "rgba(237, 129, 40, 0.89)",
258                 "value": 0
259               },
260               {
261                 "color": "#299c46",
262                 "value": 0
263               }
264             ]
265           },
266           "unit": "none"
267         },
268         "overrides": []
269       },
270       "gridPos": {
271         "h": 4,
272         "w": 6,
273         "x": 9,
274         "y": 1
275       },
276       "id": 18,
277       "links": [],
278       "maxDataPoints": 100,
279       "options": {
280         "colorMode": "value",
281         "graphMode": "none",
282         "justifyMode": "auto",
283         "orientation": "horizontal",
284         "reduceOptions": {
285           "calcs": [
286             "lastNotNull"
287           ],
288           "fields": "",
289           "values": false
290         },
291         "text": {},
292         "textMode": "auto"
293       },
294       "pluginVersion": "10.0.3",
295       "targets": [
296         {
297           "datasource": {
298             "type": "prometheus",
299             "uid": "PBFA97CFB590B2093"
300           },
301           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
302           "format": "time_series",
303           "instant": true,
304           "interval": "",
305           "intervalFactor": 1,
306           "legendFormat": "",
307           "refId": "A"
308         }
309       ],
310       "title": "Online Partitions",
311       "type": "stat"
312     },
313     {
314       "datasource": {
315         "type": "prometheus",
316         "uid": "PBFA97CFB590B2093"
317       },
318       "description": "",
319       "fieldConfig": {
320         "defaults": {
321           "color": {
322             "mode": "thresholds"
323           },
324           "mappings": [
325             {
326               "options": {
327                 "match": "null",
328                 "result": {
329                   "text": "N/A"
330                 }
331               },
332               "type": "special"
333             }
334           ],
335           "thresholds": {
336             "mode": "absolute",
337             "steps": [
338               {
339                 "color": "#299c46",
340                 "value": null
341               },
342               {
343                 "color": "rgba(237, 129, 40, 0.89)",
344                 "value": 2
345               },
346               {
347                 "color": "#d44a3a"
348               }
349             ]
350           },
351           "unit": "none"
352         },
353         "overrides": []
354       },
355       "gridPos": {
356         "h": 4,
357         "w": 4,
358         "x": 15,
359         "y": 1
360       },
361       "id": 33,
362       "links": [],
363       "maxDataPoints": 100,
364       "options": {
365         "colorMode": "value",
366         "graphMode": "none",
367         "justifyMode": "auto",
368         "orientation": "horizontal",
369         "reduceOptions": {
370           "calcs": [
371             "lastNotNull"
372           ],
373           "fields": "",
374           "values": false
375         },
376         "text": {},
377         "textMode": "auto"
378       },
379       "pluginVersion": "10.0.3",
380       "targets": [
381         {
382           "datasource": {
383             "type": "prometheus",
384             "uid": "PBFA97CFB590B2093"
385           },
386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
387           "format": "time_series",
388           "instant": true,
389           "interval": "",
390           "intervalFactor": 1,
391           "legendFormat": "",
392           "refId": "A"
393         }
394       ],
395       "title": "Preferred Replica Imbalance",
396       "type": "stat"
397     },
398     {
399       "datasource": {
400         "type": "prometheus",
401         "uid": "PBFA97CFB590B2093"
402       },
403       "description": "Unclean leader election rate",
404       "fieldConfig": {
405         "defaults": {
406           "color": {
407             "mode": "thresholds"
408           },
409           "mappings": [
410             {
411               "options": {
412                 "match": "null",
413                 "result": {
414                   "text": "N/A"
415                 }
416               },
417               "type": "special"
418             }
419           ],
420           "thresholds": {
421             "mode": "absolute",
422             "steps": [
423               {
424                 "color": "#299c46",
425                 "value": null
426               },
427               {
428                 "color": "rgba(237, 129, 40, 0.89)",
429                 "value": 2
430               },
431               {
432                 "color": "#d44a3a"
433               }
434             ]
435           },
436           "unit": "none"
437         },
438         "overrides": []
439       },
440       "gridPos": {
441         "h": 4,
442         "w": 4,
443         "x": 19,
444         "y": 1
445       },
446       "id": 16,
447       "links": [],
448       "maxDataPoints": 100,
449       "options": {
450         "colorMode": "value",
451         "graphMode": "none",
452         "justifyMode": "auto",
453         "orientation": "horizontal",
454         "reduceOptions": {
455           "calcs": [
456             "lastNotNull"
457           ],
458           "fields": "",
459           "values": false
460         },
461         "text": {},
462         "textMode": "auto"
463       },
464       "pluginVersion": "10.0.3",
465       "targets": [
466         {
467           "datasource": {
468             "type": "prometheus",
469             "uid": "PBFA97CFB590B2093"
470           },
471           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
472           "format": "time_series",
473           "instant": true,
474           "interval": "",
475           "intervalFactor": 1,
476           "legendFormat": "",
477           "refId": "A"
478         }
479       ],
480       "title": "Unclean Leader Election Rate",
481       "type": "stat"
482     },
483     {
484       "datasource": {
485         "type": "prometheus",
486         "uid": "PBFA97CFB590B2093"
487       },
488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
489       "fieldConfig": {
490         "defaults": {
491           "color": {
492             "mode": "thresholds"
493           },
494           "mappings": [
495             {
496               "options": {
497                 "match": "null",
498                 "result": {
499                   "text": "N/A"
500                 }
501               },
502               "type": "special"
503             }
504           ],
505           "thresholds": {
506             "mode": "absolute",
507             "steps": [
508               {
509                 "color": "#508642",
510                 "value": null
511               },
512               {
513                 "color": "rgba(237, 129, 40, 0.89)",
514                 "value": 1
515               },
516               {
517                 "color": "#bf1b00",
518                 "value": 5
519               }
520             ]
521           },
522           "unit": "none"
523         },
524         "overrides": []
525       },
526       "gridPos": {
527         "h": 4,
528         "w": 4,
529         "x": 0,
530         "y": 5
531       },
532       "id": 20,
533       "links": [],
534       "maxDataPoints": 100,
535       "options": {
536         "colorMode": "value",
537         "graphMode": "area",
538         "justifyMode": "auto",
539         "orientation": "horizontal",
540         "reduceOptions": {
541           "calcs": [
542             "lastNotNull"
543           ],
544           "fields": "",
545           "values": false
546         },
547         "text": {},
548         "textMode": "auto"
549       },
550       "pluginVersion": "10.0.3",
551       "targets": [
552         {
553           "datasource": {
554             "type": "prometheus",
555             "uid": "PBFA97CFB590B2093"
556           },
557           "editorMode": "code",
558           "exemplar": false,
559           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
560           "format": "time_series",
561           "hide": false,
562           "instant": false,
563           "interval": "",
564           "intervalFactor": 2,
565           "legendFormat": "",
566           "range": true,
567           "refId": "A"
568         }
569       ],
570       "title": "Under Replicated Partitions",
571       "type": "stat"
572     },
573     {
574       "datasource": {
575         "type": "prometheus",
576         "uid": "PBFA97CFB590B2093"
577       },
578       "description": "",
579       "fieldConfig": {
580         "defaults": {
581           "color": {
582             "mode": "palette-classic"
583           },
584           "custom": {
585             "axisCenteredZero": false,
586             "axisColorMode": "text",
587             "axisLabel": "",
588             "axisPlacement": "auto",
589             "barAlignment": 0,
590             "drawStyle": "line",
591             "fillOpacity": 10,
592             "gradientMode": "none",
593             "hideFrom": {
594               "legend": false,
595               "tooltip": false,
596               "viz": false
597             },
598             "lineInterpolation": "linear",
599             "lineWidth": 1,
600             "pointSize": 5,
601             "scaleDistribution": {
602               "type": "linear"
603             },
604             "showPoints": "never",
605             "spanNulls": true,
606             "stacking": {
607               "group": "A",
608               "mode": "none"
609             },
610             "thresholdsStyle": {
611               "mode": "off"
612             }
613           },
614           "decimals": 0,
615           "links": [],
616           "mappings": [
617             {
618               "options": {
619                 "null": {
620                   "index": 0,
621                   "text": "N/A"
622                 }
623               },
624               "type": "value"
625             }
626           ],
627           "min": 0,
628           "thresholds": {
629             "mode": "absolute",
630             "steps": [
631               {
632                 "color": "green",
633                 "value": null
634               },
635               {
636                 "color": "red",
637                 "value": 80
638               }
639             ]
640           },
641           "unit": "partitions"
642         },
643         "overrides": []
644       },
645       "gridPos": {
646         "h": 12,
647         "w": 11,
648         "x": 4,
649         "y": 5
650       },
651       "id": 123,
652       "options": {
653         "legend": {
654           "calcs": [
655             "last",
656             "max"
657           ],
658           "displayMode": "table",
659           "placement": "bottom",
660           "showLegend": true
661         },
662         "tooltip": {
663           "mode": "single",
664           "sort": "none"
665         }
666       },
667       "pluginVersion": "8.1.3",
668       "targets": [
669         {
670           "datasource": {
671             "type": "prometheus",
672             "uid": "PBFA97CFB590B2093"
673           },
674           "editorMode": "code",
675           "exemplar": false,
676           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
677           "hide": false,
678           "interval": "",
679           "legendFormat": "{{instance}}",
680           "range": true,
681           "refId": "A"
682         }
683       ],
684       "title": "Underreplicated Partitions",
685       "type": "timeseries"
686     },
687     {
688       "datasource": {
689         "type": "prometheus",
690         "uid": "PBFA97CFB590B2093"
691       },
692       "description": "",
693       "fieldConfig": {
694         "defaults": {
695           "color": {
696             "mode": "palette-classic"
697           },
698           "custom": {
699             "hideFrom": {
700               "legend": false,
701               "tooltip": false,
702               "viz": false
703             }
704           },
705           "decimals": 0,
706           "links": [],
707           "mappings": [
708             {
709               "options": {
710                 "null": {
711                   "index": 0,
712                   "text": "0"
713                 }
714               },
715               "type": "value"
716             }
717           ],
718           "min": 0,
719           "unit": "partitions"
720         },
721         "overrides": []
722       },
723       "gridPos": {
724         "h": 12,
725         "w": 8,
726         "x": 15,
727         "y": 5
728       },
729       "id": 124,
730       "options": {
731         "displayLabels": [
732           "percent"
733         ],
734         "legend": {
735           "displayMode": "list",
736           "placement": "right",
737           "showLegend": true,
738           "values": [
739             "value"
740           ]
741         },
742         "pieType": "pie",
743         "reduceOptions": {
744           "calcs": [
745             "last"
746           ],
747           "fields": "",
748           "values": false
749         },
750         "tooltip": {
751           "mode": "single",
752           "sort": "none"
753         }
754       },
755       "pluginVersion": "8.1.3",
756       "targets": [
757         {
758           "datasource": {
759             "type": "prometheus",
760             "uid": "PBFA97CFB590B2093"
761           },
762           "editorMode": "code",
763           "exemplar": false,
764           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
765           "hide": false,
766           "interval": "",
767           "legendFormat": "{{instance}}",
768           "range": true,
769           "refId": "A"
770         }
771       ],
772       "title": "Leader Count",
773       "type": "piechart"
774     },
775     {
776       "datasource": {
777         "type": "prometheus",
778         "uid": "PBFA97CFB590B2093"
779       },
780       "description": "Number of partitions under min insync replicas.",
781       "fieldConfig": {
782         "defaults": {
783           "color": {
784             "mode": "thresholds"
785           },
786           "mappings": [
787             {
788               "options": {
789                 "match": "null",
790                 "result": {
791                   "text": "N/A"
792                 }
793               },
794               "type": "special"
795             }
796           ],
797           "thresholds": {
798             "mode": "absolute",
799             "steps": [
800               {
801                 "color": "#508642",
802                 "value": null
803               },
804               {
805                 "color": "rgba(237, 129, 40, 0.89)",
806                 "value": 1
807               },
808               {
809                 "color": "#bf1b00",
810                 "value": 5
811               }
812             ]
813           },
814           "unit": "none"
815         },
816         "overrides": []
817       },
818       "gridPos": {
819         "h": 4,
820         "w": 4,
821         "x": 0,
822         "y": 9
823       },
824       "id": 32,
825       "links": [],
826       "maxDataPoints": 100,
827       "options": {
828         "colorMode": "value",
829         "graphMode": "area",
830         "justifyMode": "auto",
831         "orientation": "horizontal",
832         "reduceOptions": {
833           "calcs": [
834             "lastNotNull"
835           ],
836           "fields": "",
837           "values": false
838         },
839         "text": {},
840         "textMode": "auto"
841       },
842       "pluginVersion": "10.0.3",
843       "targets": [
844         {
845           "datasource": {
846             "type": "prometheus",
847             "uid": "PBFA97CFB590B2093"
848           },
849           "editorMode": "code",
850           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
851           "format": "time_series",
852           "hide": false,
853           "instant": false,
854           "interval": "",
855           "intervalFactor": 2,
856           "legendFormat": "",
857           "range": true,
858           "refId": "A"
859         }
860       ],
861       "title": "Under Min ISR Partitions",
862       "type": "stat"
863     },
864     {
865       "datasource": {
866         "type": "prometheus",
867         "uid": "PBFA97CFB590B2093"
868       },
869       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
870       "fieldConfig": {
871         "defaults": {
872           "color": {
873             "mode": "thresholds"
874           },
875           "mappings": [
876             {
877               "options": {
878                 "match": "null",
879                 "result": {
880                   "text": "N/A"
881                 }
882               },
883               "type": "special"
884             }
885           ],
886           "thresholds": {
887             "mode": "absolute",
888             "steps": [
889               {
890                 "color": "#508642",
891                 "value": null
892               },
893               {
894                 "color": "#ef843c",
895                 "value": 1
896               },
897               {
898                 "color": "#bf1b00",
899                 "value": 1
900               }
901             ]
902           },
903           "unit": "none"
904         },
905         "overrides": []
906       },
907       "gridPos": {
908         "h": 4,
909         "w": 4,
910         "x": 0,
911         "y": 13
912       },
913       "id": 22,
914       "links": [],
915       "maxDataPoints": 100,
916       "options": {
917         "colorMode": "value",
918         "graphMode": "area",
919         "justifyMode": "auto",
920         "orientation": "horizontal",
921         "reduceOptions": {
922           "calcs": [
923             "lastNotNull"
924           ],
925           "fields": "",
926           "values": false
927         },
928         "text": {},
929         "textMode": "auto"
930       },
931       "pluginVersion": "10.0.3",
932       "targets": [
933         {
934           "datasource": {
935             "type": "prometheus",
936             "uid": "PBFA97CFB590B2093"
937           },
938           "editorMode": "code",
939           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
940           "format": "time_series",
941           "instant": false,
942           "interval": "",
943           "intervalFactor": 1,
944           "legendFormat": "",
945           "range": true,
946           "refId": "A"
947         }
948       ],
949       "title": "Offline Partitions Count",
950       "type": "stat"
951     },
952     {
953       "collapsed": true,
954       "datasource": {
955         "type": "datasource",
956         "uid": "grafana"
957       },
958       "gridPos": {
959         "h": 1,
960         "w": 24,
961         "x": 0,
962         "y": 17
963       },
964       "id": 40,
965       "panels": [
966         {
967           "datasource": {
968             "type": "prometheus",
969             "uid": "PBFA97CFB590B2093"
970           },
971           "fieldConfig": {
972             "defaults": {
973               "color": {
974                 "mode": "palette-classic"
975               },
976               "custom": {
977                 "axisCenteredZero": false,
978                 "axisColorMode": "text",
979                 "axisLabel": "Cores",
980                 "axisPlacement": "auto",
981                 "barAlignment": 0,
982                 "drawStyle": "line",
983                 "fillOpacity": 10,
984                 "gradientMode": "none",
985                 "hideFrom": {
986                   "legend": false,
987                   "tooltip": false,
988                   "viz": false
989                 },
990                 "lineInterpolation": "linear",
991                 "lineWidth": 2,
992                 "pointSize": 5,
993                 "scaleDistribution": {
994                   "type": "linear"
995                 },
996                 "showPoints": "never",
997                 "spanNulls": false,
998                 "stacking": {
999                   "group": "A",
1000                   "mode": "none"
1001                 },
1002                 "thresholdsStyle": {
1003                   "mode": "off"
1004                 }
1005               },
1006               "links": [],
1007               "mappings": [],
1008               "thresholds": {
1009                 "mode": "absolute",
1010                 "steps": [
1011                   {
1012                     "color": "green"
1013                   },
1014                   {
1015                     "color": "red",
1016                     "value": 80
1017                   }
1018                 ]
1019               },
1020               "unit": "percent"
1021             },
1022             "overrides": [
1023               {
1024                 "matcher": {
1025                   "id": "byName",
1026                   "options": "localhost:7071"
1027                 },
1028                 "properties": [
1029                   {
1030                     "id": "color",
1031                     "value": {
1032                       "fixedColor": "#629E51",
1033                       "mode": "fixed"
1034                     }
1035                   }
1036                 ]
1037               }
1038             ]
1039           },
1040           "gridPos": {
1041             "h": 12,
1042             "w": 8,
1043             "x": 0,
1044             "y": 19
1045           },
1046           "id": 27,
1047           "links": [],
1048           "options": {
1049             "legend": {
1050               "calcs": [
1051                 "mean",
1052                 "lastNotNull",
1053                 "max"
1054               ],
1055               "displayMode": "table",
1056               "placement": "bottom",
1057               "showLegend": true
1058             },
1059             "tooltip": {
1060               "mode": "single",
1061               "sort": "none"
1062             }
1063           },
1064           "pluginVersion": "8.1.3",
1065           "targets": [
1066             {
1067               "datasource": {
1068                 "type": "prometheus",
1069                 "uid": "PBFA97CFB590B2093"
1070               },
1071               "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1072               "format": "time_series",
1073               "interval": "",
1074               "intervalFactor": 2,
1075               "legendFormat": "{{instance}}",
1076               "metric": "process_cpu_secondspersec",
1077               "refId": "A",
1078               "step": 4
1079             }
1080           ],
1081           "title": "CPU Usage",
1082           "type": "timeseries"
1083         },
1084         {
1085           "datasource": {
1086             "type": "prometheus",
1087             "uid": "PBFA97CFB590B2093"
1088           },
1089           "fieldConfig": {
1090             "defaults": {
1091               "color": {
1092                 "mode": "palette-classic"
1093               },
1094               "custom": {
1095                 "axisCenteredZero": false,
1096                 "axisColorMode": "text",
1097                 "axisLabel": "Memory",
1098                 "axisPlacement": "auto",
1099                 "barAlignment": 0,
1100                 "drawStyle": "line",
1101                 "fillOpacity": 10,
1102                 "gradientMode": "none",
1103                 "hideFrom": {
1104                   "legend": false,
1105                   "tooltip": false,
1106                   "viz": false
1107                 },
1108                 "lineInterpolation": "linear",
1109                 "lineWidth": 2,
1110                 "pointSize": 5,
1111                 "scaleDistribution": {
1112                   "type": "linear"
1113                 },
1114                 "showPoints": "never",
1115                 "spanNulls": false,
1116                 "stacking": {
1117                   "group": "A",
1118                   "mode": "none"
1119                 },
1120                 "thresholdsStyle": {
1121                   "mode": "off"
1122                 }
1123               },
1124               "links": [],
1125               "mappings": [],
1126               "thresholds": {
1127                 "mode": "absolute",
1128                 "steps": [
1129                   {
1130                     "color": "green"
1131                   },
1132                   {
1133                     "color": "red",
1134                     "value": 80
1135                   }
1136                 ]
1137               },
1138               "unit": "bytes"
1139             },
1140             "overrides": [
1141               {
1142                 "matcher": {
1143                   "id": "byName",
1144                   "options": "localhost:7071"
1145                 },
1146                 "properties": [
1147                   {
1148                     "id": "color",
1149                     "value": {
1150                       "fixedColor": "#BA43A9",
1151                       "mode": "fixed"
1152                     }
1153                   }
1154                 ]
1155               },
1156               {
1157                 "__systemRef": "hideSeriesFrom",
1158                 "matcher": {
1159                   "id": "byNames",
1160                   "options": {
1161                     "mode": "exclude",
1162                     "names": [
1163                       "kafka-2:80"
1164                     ],
1165                     "prefix": "All except:",
1166                     "readOnly": true
1167                   }
1168                 },
1169                 "properties": [
1170                   {
1171                     "id": "custom.hideFrom",
1172                     "value": {
1173                       "legend": false,
1174                       "tooltip": false,
1175                       "viz": true
1176                     }
1177                   }
1178                 ]
1179               }
1180             ]
1181           },
1182           "gridPos": {
1183             "h": 12,
1184             "w": 8,
1185             "x": 8,
1186             "y": 19
1187           },
1188           "id": 2,
1189           "links": [],
1190           "options": {
1191             "legend": {
1192               "calcs": [
1193                 "mean",
1194                 "lastNotNull",
1195                 "max"
1196               ],
1197               "displayMode": "table",
1198               "placement": "bottom",
1199               "showLegend": true
1200             },
1201             "tooltip": {
1202               "mode": "single",
1203               "sort": "none"
1204             }
1205           },
1206           "pluginVersion": "8.1.3",
1207           "targets": [
1208             {
1209               "datasource": {
1210                 "type": "prometheus",
1211                 "uid": "PBFA97CFB590B2093"
1212               },
1213               "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1214               "interval": "",
1215               "intervalFactor": 2,
1216               "legendFormat": "{{instance}}",
1217               "metric": "jvm_memory_bytes_used",
1218               "refId": "A",
1219               "step": 4
1220             },
1221             {
1222               "datasource": {
1223                 "type": "prometheus",
1224                 "uid": "PBFA97CFB590B2093"
1225               },
1226               "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1227               "interval": "",
1228               "legendFormat": "{{instance}}",
1229               "refId": "B"
1230             }
1231           ],
1232           "title": "JVM Memory Used",
1233           "type": "timeseries"
1234         },
1235         {
1236           "datasource": {
1237             "type": "prometheus",
1238             "uid": "PBFA97CFB590B2093"
1239           },
1240           "fieldConfig": {
1241             "defaults": {
1242               "color": {
1243                 "mode": "palette-classic"
1244               },
1245               "custom": {
1246                 "axisCenteredZero": false,
1247                 "axisColorMode": "text",
1248                 "axisLabel": "% time in GC",
1249                 "axisPlacement": "auto",
1250                 "barAlignment": 0,
1251                 "drawStyle": "line",
1252                 "fillOpacity": 10,
1253                 "gradientMode": "none",
1254                 "hideFrom": {
1255                   "legend": false,
1256                   "tooltip": false,
1257                   "viz": false
1258                 },
1259                 "lineInterpolation": "linear",
1260                 "lineWidth": 2,
1261                 "pointSize": 5,
1262                 "scaleDistribution": {
1263                   "type": "linear"
1264                 },
1265                 "showPoints": "never",
1266                 "spanNulls": false,
1267                 "stacking": {
1268                   "group": "A",
1269                   "mode": "none"
1270                 },
1271                 "thresholdsStyle": {
1272                   "mode": "off"
1273                 }
1274               },
1275               "links": [],
1276               "mappings": [],
1277               "thresholds": {
1278                 "mode": "absolute",
1279                 "steps": [
1280                   {
1281                     "color": "green"
1282                   },
1283                   {
1284                     "color": "red",
1285                     "value": 80
1286                   }
1287                 ]
1288               },
1289               "unit": "percentunit"
1290             },
1291             "overrides": [
1292               {
1293                 "matcher": {
1294                   "id": "byName",
1295                   "options": "localhost:7071"
1296                 },
1297                 "properties": [
1298                   {
1299                     "id": "color",
1300                     "value": {
1301                       "fixedColor": "#890F02",
1302                       "mode": "fixed"
1303                     }
1304                   }
1305                 ]
1306               }
1307             ]
1308           },
1309           "gridPos": {
1310             "h": 12,
1311             "w": 8,
1312             "x": 16,
1313             "y": 19
1314           },
1315           "id": 3,
1316           "links": [],
1317           "options": {
1318             "legend": {
1319               "calcs": [
1320                 "mean",
1321                 "lastNotNull",
1322                 "max"
1323               ],
1324               "displayMode": "table",
1325               "placement": "bottom",
1326               "showLegend": true
1327             },
1328             "tooltip": {
1329               "mode": "single",
1330               "sort": "none"
1331             }
1332           },
1333           "pluginVersion": "8.1.3",
1334           "targets": [
1335             {
1336               "datasource": {
1337                 "type": "prometheus",
1338                 "uid": "PBFA97CFB590B2093"
1339               },
1340               "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1341               "interval": "",
1342               "intervalFactor": 2,
1343               "legendFormat": "{{instance}}",
1344               "metric": "jvm_gc_collection_seconds_sum",
1345               "refId": "A",
1346               "step": 4
1347             }
1348           ],
1349           "title": "Time spent in GC",
1350           "type": "timeseries"
1351         }
1352       ],
1353       "targets": [
1354         {
1355           "datasource": {
1356             "type": "datasource",
1357             "uid": "grafana"
1358           },
1359           "refId": "A"
1360         }
1361       ],
1362       "title": "System",
1363       "type": "row"
1364     },
1365     {
1366       "collapsed": false,
1367       "datasource": {
1368         "type": "datasource",
1369         "uid": "grafana"
1370       },
1371       "gridPos": {
1372         "h": 1,
1373         "w": 24,
1374         "x": 0,
1375         "y": 18
1376       },
1377       "id": 31,
1378       "panels": [],
1379       "targets": [
1380         {
1381           "datasource": {
1382             "type": "datasource",
1383             "uid": "grafana"
1384           },
1385           "refId": "A"
1386         }
1387       ],
1388       "title": "Throughput & Request rate",
1389       "type": "row"
1390     },
1391     {
1392       "datasource": {
1393         "type": "prometheus",
1394         "uid": "PBFA97CFB590B2093"
1395       },
1396       "fieldConfig": {
1397         "defaults": {
1398           "color": {
1399             "mode": "palette-classic"
1400           },
1401           "custom": {
1402             "axisCenteredZero": false,
1403             "axisColorMode": "text",
1404             "axisLabel": "Bytes/s",
1405             "axisPlacement": "auto",
1406             "barAlignment": 0,
1407             "drawStyle": "line",
1408             "fillOpacity": 10,
1409             "gradientMode": "none",
1410             "hideFrom": {
1411               "legend": false,
1412               "tooltip": false,
1413               "viz": false
1414             },
1415             "lineInterpolation": "linear",
1416             "lineWidth": 2,
1417             "pointSize": 5,
1418             "scaleDistribution": {
1419               "type": "linear"
1420             },
1421             "showPoints": "never",
1422             "spanNulls": false,
1423             "stacking": {
1424               "group": "A",
1425               "mode": "none"
1426             },
1427             "thresholdsStyle": {
1428               "mode": "off"
1429             }
1430           },
1431           "links": [],
1432           "mappings": [],
1433           "thresholds": {
1434             "mode": "absolute",
1435             "steps": [
1436               {
1437                 "color": "green",
1438                 "value": null
1439               },
1440               {
1441                 "color": "red",
1442                 "value": 80
1443               }
1444             ]
1445           },
1446           "unit": "Bps"
1447         },
1448         "overrides": []
1449       },
1450       "gridPos": {
1451         "h": 8,
1452         "w": 12,
1453         "x": 0,
1454         "y": 19
1455       },
1456       "id": 84,
1457       "links": [],
1458       "options": {
1459         "legend": {
1460           "calcs": [
1461             "mean",
1462             "lastNotNull",
1463             "max"
1464           ],
1465           "displayMode": "list",
1466           "placement": "bottom",
1467           "showLegend": true
1468         },
1469         "tooltip": {
1470           "mode": "single",
1471           "sort": "none"
1472         }
1473       },
1474       "pluginVersion": "8.1.3",
1475       "targets": [
1476         {
1477           "datasource": {
1478             "type": "prometheus",
1479             "uid": "PBFA97CFB590B2093"
1480           },
1481           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1482           "format": "time_series",
1483           "interval": "",
1484           "intervalFactor": 2,
1485           "legendFormat": "Bytes in",
1486           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1487           "refId": "A",
1488           "step": 4
1489         },
1490         {
1491           "datasource": {
1492             "type": "prometheus",
1493             "uid": "PBFA97CFB590B2093"
1494           },
1495           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1496           "format": "time_series",
1497           "hide": false,
1498           "interval": "",
1499           "intervalFactor": 2,
1500           "legendFormat": "Bytes out",
1501           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1502           "refId": "B",
1503           "step": 4
1504         }
1505       ],
1506       "title": "Broker network throughput",
1507       "type": "timeseries"
1508     },
1509     {
1510       "datasource": {
1511         "type": "prometheus",
1512         "uid": "PBFA97CFB590B2093"
1513       },
1514       "description": "Produce request rate.",
1515       "fieldConfig": {
1516         "defaults": {
1517           "mappings": [
1518             {
1519               "options": {
1520                 "match": "null",
1521                 "result": {
1522                   "text": "N/A"
1523                 }
1524               },
1525               "type": "special"
1526             }
1527           ],
1528           "thresholds": {
1529             "mode": "absolute",
1530             "steps": [
1531               {
1532                 "color": "green",
1533                 "value": null
1534               }
1535             ]
1536           },
1537           "unit": "none"
1538         },
1539         "overrides": []
1540       },
1541       "gridPos": {
1542         "h": 4,
1543         "w": 4,
1544         "x": 12,
1545         "y": 19
1546       },
1547       "id": 93,
1548       "links": [],
1549       "maxDataPoints": 100,
1550       "options": {
1551         "colorMode": "value",
1552         "graphMode": "area",
1553         "justifyMode": "auto",
1554         "orientation": "horizontal",
1555         "reduceOptions": {
1556           "calcs": [
1557             "lastNotNull"
1558           ],
1559           "fields": "",
1560           "values": false
1561         },
1562         "text": {},
1563         "textMode": "auto"
1564       },
1565       "pluginVersion": "10.0.3",
1566       "targets": [
1567         {
1568           "datasource": {
1569             "type": "prometheus",
1570             "uid": "PBFA97CFB590B2093"
1571           },
1572           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1573           "interval": "",
1574           "legendFormat": "",
1575           "refId": "A"
1576         }
1577       ],
1578       "title": "All Request Per Sec",
1579       "type": "stat"
1580     },
1581     {
1582       "datasource": {
1583         "type": "prometheus",
1584         "uid": "PBFA97CFB590B2093"
1585       },
1586       "description": "Fetch request rate.",
1587       "fieldConfig": {
1588         "defaults": {
1589           "mappings": [
1590             {
1591               "options": {
1592                 "match": "null",
1593                 "result": {
1594                   "text": "N/A"
1595                 }
1596               },
1597               "type": "special"
1598             }
1599           ],
1600           "thresholds": {
1601             "mode": "absolute",
1602             "steps": [
1603               {
1604                 "color": "green",
1605                 "value": null
1606               }
1607             ]
1608           },
1609           "unit": "none"
1610         },
1611         "overrides": []
1612       },
1613       "gridPos": {
1614         "h": 4,
1615         "w": 4,
1616         "x": 16,
1617         "y": 19
1618       },
1619       "id": 37,
1620       "links": [],
1621       "maxDataPoints": 100,
1622       "options": {
1623         "colorMode": "value",
1624         "graphMode": "area",
1625         "justifyMode": "auto",
1626         "orientation": "horizontal",
1627         "reduceOptions": {
1628           "calcs": [
1629             "lastNotNull"
1630           ],
1631           "fields": "",
1632           "values": false
1633         },
1634         "text": {},
1635         "textMode": "auto"
1636       },
1637       "pluginVersion": "10.0.3",
1638       "targets": [
1639         {
1640           "datasource": {
1641             "type": "prometheus",
1642             "uid": "PBFA97CFB590B2093"
1643           },
1644           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1645           "interval": "",
1646           "legendFormat": "",
1647           "refId": "A"
1648         }
1649       ],
1650       "title": "Consumer Fetch Request Per Sec",
1651       "type": "stat"
1652     },
1653     {
1654       "datasource": {
1655         "type": "prometheus",
1656         "uid": "PBFA97CFB590B2093"
1657       },
1658       "description": "Produce request rate.",
1659       "fieldConfig": {
1660         "defaults": {
1661           "mappings": [
1662             {
1663               "options": {
1664                 "match": "null",
1665                 "result": {
1666                   "text": "N/A"
1667                 }
1668               },
1669               "type": "special"
1670             }
1671           ],
1672           "thresholds": {
1673             "mode": "absolute",
1674             "steps": [
1675               {
1676                 "color": "green",
1677                 "value": null
1678               }
1679             ]
1680           },
1681           "unit": "none"
1682         },
1683         "overrides": []
1684       },
1685       "gridPos": {
1686         "h": 4,
1687         "w": 4,
1688         "x": 20,
1689         "y": 19
1690       },
1691       "id": 35,
1692       "links": [],
1693       "maxDataPoints": 100,
1694       "options": {
1695         "colorMode": "value",
1696         "graphMode": "area",
1697         "justifyMode": "auto",
1698         "orientation": "horizontal",
1699         "reduceOptions": {
1700           "calcs": [
1701             "lastNotNull"
1702           ],
1703           "fields": "",
1704           "values": false
1705         },
1706         "text": {},
1707         "textMode": "auto"
1708       },
1709       "pluginVersion": "10.0.3",
1710       "targets": [
1711         {
1712           "datasource": {
1713             "type": "prometheus",
1714             "uid": "PBFA97CFB590B2093"
1715           },
1716           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1717           "interval": "",
1718           "legendFormat": "",
1719           "refId": "A"
1720         }
1721       ],
1722       "title": "Produce Request Per Sec",
1723       "type": "stat"
1724     },
1725     {
1726       "datasource": {
1727         "type": "prometheus",
1728         "uid": "PBFA97CFB590B2093"
1729       },
1730       "description": "Metadata request rate.",
1731       "fieldConfig": {
1732         "defaults": {
1733           "mappings": [
1734             {
1735               "options": {
1736                 "match": "null",
1737                 "result": {
1738                   "text": "N/A"
1739                 }
1740               },
1741               "type": "special"
1742             }
1743           ],
1744           "thresholds": {
1745             "mode": "absolute",
1746             "steps": [
1747               {
1748                 "color": "green",
1749                 "value": null
1750               }
1751             ]
1752           },
1753           "unit": "none"
1754         },
1755         "overrides": []
1756       },
1757       "gridPos": {
1758         "h": 4,
1759         "w": 4,
1760         "x": 12,
1761         "y": 23
1762       },
1763       "id": 36,
1764       "links": [],
1765       "maxDataPoints": 100,
1766       "options": {
1767         "colorMode": "value",
1768         "graphMode": "area",
1769         "justifyMode": "auto",
1770         "orientation": "horizontal",
1771         "reduceOptions": {
1772           "calcs": [
1773             "lastNotNull"
1774           ],
1775           "fields": "",
1776           "values": false
1777         },
1778         "text": {},
1779         "textMode": "auto"
1780       },
1781       "pluginVersion": "10.0.3",
1782       "targets": [
1783         {
1784           "datasource": {
1785             "type": "prometheus",
1786             "uid": "PBFA97CFB590B2093"
1787           },
1788           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1789           "interval": "",
1790           "legendFormat": "",
1791           "refId": "A"
1792         }
1793       ],
1794       "title": "Metadata Request Per Sec",
1795       "type": "stat"
1796     },
1797     {
1798       "datasource": {
1799         "type": "prometheus",
1800         "uid": "PBFA97CFB590B2093"
1801       },
1802       "description": "Offset Commit request rate.",
1803       "fieldConfig": {
1804         "defaults": {
1805           "mappings": [
1806             {
1807               "options": {
1808                 "match": "null",
1809                 "result": {
1810                   "text": "N/A"
1811                 }
1812               },
1813               "type": "special"
1814             }
1815           ],
1816           "thresholds": {
1817             "mode": "absolute",
1818             "steps": [
1819               {
1820                 "color": "green",
1821                 "value": null
1822               }
1823             ]
1824           },
1825           "unit": "none"
1826         },
1827         "overrides": []
1828       },
1829       "gridPos": {
1830         "h": 4,
1831         "w": 4,
1832         "x": 16,
1833         "y": 23
1834       },
1835       "id": 38,
1836       "links": [],
1837       "maxDataPoints": 100,
1838       "options": {
1839         "colorMode": "value",
1840         "graphMode": "area",
1841         "justifyMode": "auto",
1842         "orientation": "horizontal",
1843         "reduceOptions": {
1844           "calcs": [
1845             "lastNotNull"
1846           ],
1847           "fields": "",
1848           "values": false
1849         },
1850         "text": {},
1851         "textMode": "auto"
1852       },
1853       "pluginVersion": "10.0.3",
1854       "targets": [
1855         {
1856           "datasource": {
1857             "type": "prometheus",
1858             "uid": "PBFA97CFB590B2093"
1859           },
1860           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1861           "interval": "",
1862           "legendFormat": "",
1863           "refId": "A"
1864         }
1865       ],
1866       "title": "Offset Commit Request Per Sec",
1867       "type": "stat"
1868     },
1869     {
1870       "datasource": {
1871         "type": "prometheus",
1872         "uid": "PBFA97CFB590B2093"
1873       },
1874       "description": "Fetch request rate.",
1875       "fieldConfig": {
1876         "defaults": {
1877           "mappings": [
1878             {
1879               "options": {
1880                 "match": "null",
1881                 "result": {
1882                   "text": "N/A"
1883                 }
1884               },
1885               "type": "special"
1886             }
1887           ],
1888           "thresholds": {
1889             "mode": "absolute",
1890             "steps": [
1891               {
1892                 "color": "green",
1893                 "value": null
1894               }
1895             ]
1896           },
1897           "unit": "none"
1898         },
1899         "overrides": []
1900       },
1901       "gridPos": {
1902         "h": 4,
1903         "w": 4,
1904         "x": 20,
1905         "y": 23
1906       },
1907       "id": 94,
1908       "links": [],
1909       "maxDataPoints": 100,
1910       "options": {
1911         "colorMode": "value",
1912         "graphMode": "area",
1913         "justifyMode": "auto",
1914         "orientation": "horizontal",
1915         "reduceOptions": {
1916           "calcs": [
1917             "lastNotNull"
1918           ],
1919           "fields": "",
1920           "values": false
1921         },
1922         "text": {},
1923         "textMode": "auto"
1924       },
1925       "pluginVersion": "10.0.3",
1926       "targets": [
1927         {
1928           "datasource": {
1929             "type": "prometheus",
1930             "uid": "PBFA97CFB590B2093"
1931           },
1932           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1933           "interval": "",
1934           "legendFormat": "",
1935           "refId": "A"
1936         }
1937       ],
1938       "title": "Broker Fetch Request Per Sec",
1939       "type": "stat"
1940     },
1941     {
1942       "collapsed": false,
1943       "datasource": {
1944         "type": "datasource",
1945         "uid": "grafana"
1946       },
1947       "gridPos": {
1948         "h": 1,
1949         "w": 24,
1950         "x": 0,
1951         "y": 27
1952       },
1953       "id": 29,
1954       "panels": [],
1955       "targets": [
1956         {
1957           "datasource": {
1958             "type": "datasource",
1959             "uid": "grafana"
1960           },
1961           "refId": "A"
1962         }
1963       ],
1964       "title": "Throughput In/Out",
1965       "type": "row"
1966     },
1967     {
1968       "datasource": {
1969         "type": "prometheus",
1970         "uid": "PBFA97CFB590B2093"
1971       },
1972       "fieldConfig": {
1973         "defaults": {
1974           "color": {
1975             "mode": "palette-classic"
1976           },
1977           "custom": {
1978             "axisCenteredZero": false,
1979             "axisColorMode": "text",
1980             "axisLabel": "Messages/s",
1981             "axisPlacement": "auto",
1982             "barAlignment": 0,
1983             "drawStyle": "line",
1984             "fillOpacity": 10,
1985             "gradientMode": "none",
1986             "hideFrom": {
1987               "legend": false,
1988               "tooltip": false,
1989               "viz": false
1990             },
1991             "lineInterpolation": "linear",
1992             "lineWidth": 2,
1993             "pointSize": 5,
1994             "scaleDistribution": {
1995               "type": "linear"
1996             },
1997             "showPoints": "never",
1998             "spanNulls": false,
1999             "stacking": {
2000               "group": "A",
2001               "mode": "none"
2002             },
2003             "thresholdsStyle": {
2004               "mode": "off"
2005             }
2006           },
2007           "links": [],
2008           "mappings": [],
2009           "min": 0,
2010           "thresholds": {
2011             "mode": "absolute",
2012             "steps": [
2013               {
2014                 "color": "green",
2015                 "value": null
2016               }
2017             ]
2018           },
2019           "unit": "cps"
2020         },
2021         "overrides": []
2022       },
2023       "gridPos": {
2024         "h": 8,
2025         "w": 8,
2026         "x": 0,
2027         "y": 28
2028       },
2029       "id": 4,
2030       "links": [],
2031       "options": {
2032         "legend": {
2033           "calcs": [
2034             "mean",
2035             "lastNotNull",
2036             "max"
2037           ],
2038           "displayMode": "table",
2039           "placement": "bottom",
2040           "showLegend": true
2041         },
2042         "tooltip": {
2043           "mode": "single",
2044           "sort": "none"
2045         }
2046       },
2047       "pluginVersion": "8.1.3",
2048       "targets": [
2049         {
2050           "datasource": {
2051             "type": "prometheus",
2052             "uid": "PBFA97CFB590B2093"
2053           },
2054           "exemplar": true,
2055           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2056           "interval": "",
2057           "intervalFactor": 2,
2058           "legendFormat": "bytes/sec",
2059           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2060           "refId": "A",
2061           "step": 4
2062         }
2063       ],
2064       "title": "Messages In",
2065       "type": "timeseries"
2066     },
2067     {
2068       "datasource": {
2069         "type": "prometheus",
2070         "uid": "PBFA97CFB590B2093"
2071       },
2072       "fieldConfig": {
2073         "defaults": {
2074           "color": {
2075             "mode": "palette-classic"
2076           },
2077           "custom": {
2078             "axisCenteredZero": false,
2079             "axisColorMode": "text",
2080             "axisLabel": "Bytes/s",
2081             "axisPlacement": "auto",
2082             "barAlignment": 0,
2083             "drawStyle": "line",
2084             "fillOpacity": 10,
2085             "gradientMode": "none",
2086             "hideFrom": {
2087               "legend": false,
2088               "tooltip": false,
2089               "viz": false
2090             },
2091             "lineInterpolation": "linear",
2092             "lineWidth": 2,
2093             "pointSize": 5,
2094             "scaleDistribution": {
2095               "type": "linear"
2096             },
2097             "showPoints": "never",
2098             "spanNulls": false,
2099             "stacking": {
2100               "group": "A",
2101               "mode": "none"
2102             },
2103             "thresholdsStyle": {
2104               "mode": "off"
2105             }
2106           },
2107           "links": [],
2108           "mappings": [],
2109           "min": 0,
2110           "thresholds": {
2111             "mode": "absolute",
2112             "steps": [
2113               {
2114                 "color": "green",
2115                 "value": null
2116               },
2117               {
2118                 "color": "red",
2119                 "value": 80
2120               }
2121             ]
2122           },
2123           "unit": "Bps"
2124         },
2125         "overrides": []
2126       },
2127       "gridPos": {
2128         "h": 8,
2129         "w": 8,
2130         "x": 8,
2131         "y": 28
2132       },
2133       "id": 5,
2134       "links": [],
2135       "options": {
2136         "legend": {
2137           "calcs": [
2138             "mean",
2139             "lastNotNull",
2140             "max"
2141           ],
2142           "displayMode": "table",
2143           "placement": "bottom",
2144           "showLegend": true
2145         },
2146         "tooltip": {
2147           "mode": "single",
2148           "sort": "none"
2149         }
2150       },
2151       "pluginVersion": "8.1.3",
2152       "targets": [
2153         {
2154           "datasource": {
2155             "type": "prometheus",
2156             "uid": "PBFA97CFB590B2093"
2157           },
2158           "exemplar": true,
2159           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2160           "format": "time_series",
2161           "interval": "",
2162           "intervalFactor": 2,
2163           "legendFormat": "bytes/sec",
2164           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2165           "refId": "A",
2166           "step": 4
2167         }
2168       ],
2169       "title": "Bytes In",
2170       "type": "timeseries"
2171     },
2172     {
2173       "datasource": {
2174         "type": "prometheus",
2175         "uid": "PBFA97CFB590B2093"
2176       },
2177       "fieldConfig": {
2178         "defaults": {
2179           "color": {
2180             "mode": "palette-classic"
2181           },
2182           "custom": {
2183             "axisCenteredZero": false,
2184             "axisColorMode": "text",
2185             "axisLabel": "Bytes/s",
2186             "axisPlacement": "auto",
2187             "barAlignment": 0,
2188             "drawStyle": "line",
2189             "fillOpacity": 10,
2190             "gradientMode": "none",
2191             "hideFrom": {
2192               "legend": false,
2193               "tooltip": false,
2194               "viz": false
2195             },
2196             "lineInterpolation": "linear",
2197             "lineWidth": 2,
2198             "pointSize": 5,
2199             "scaleDistribution": {
2200               "type": "linear"
2201             },
2202             "showPoints": "never",
2203             "spanNulls": false,
2204             "stacking": {
2205               "group": "A",
2206               "mode": "none"
2207             },
2208             "thresholdsStyle": {
2209               "mode": "off"
2210             }
2211           },
2212           "links": [],
2213           "mappings": [],
2214           "min": 0,
2215           "thresholds": {
2216             "mode": "absolute",
2217             "steps": [
2218               {
2219                 "color": "green",
2220                 "value": null
2221               },
2222               {
2223                 "color": "red",
2224                 "value": 80
2225               }
2226             ]
2227           },
2228           "unit": "Bps"
2229         },
2230         "overrides": []
2231       },
2232       "gridPos": {
2233         "h": 8,
2234         "w": 8,
2235         "x": 16,
2236         "y": 28
2237       },
2238       "id": 6,
2239       "links": [],
2240       "options": {
2241         "legend": {
2242           "calcs": [
2243             "mean",
2244             "lastNotNull",
2245             "max"
2246           ],
2247           "displayMode": "table",
2248           "placement": "bottom",
2249           "showLegend": true
2250         },
2251         "tooltip": {
2252           "mode": "single",
2253           "sort": "none"
2254         }
2255       },
2256       "pluginVersion": "8.1.3",
2257       "targets": [
2258         {
2259           "datasource": {
2260             "type": "prometheus",
2261             "uid": "PBFA97CFB590B2093"
2262           },
2263           "exemplar": true,
2264           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2265           "interval": "",
2266           "intervalFactor": 2,
2267           "legendFormat": "bytes/sec",
2268           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2269           "refId": "A",
2270           "step": 4
2271         }
2272       ],
2273       "title": "Bytes Out",
2274       "type": "timeseries"
2275     },
2276     {
2277       "datasource": {
2278         "type": "prometheus",
2279         "uid": "PBFA97CFB590B2093"
2280       },
2281       "fieldConfig": {
2282         "defaults": {
2283           "color": {
2284             "mode": "palette-classic"
2285           },
2286           "custom": {
2287             "axisCenteredZero": false,
2288             "axisColorMode": "text",
2289             "axisLabel": "Messages/s",
2290             "axisPlacement": "auto",
2291             "barAlignment": 0,
2292             "drawStyle": "line",
2293             "fillOpacity": 10,
2294             "gradientMode": "none",
2295             "hideFrom": {
2296               "legend": false,
2297               "tooltip": false,
2298               "viz": false
2299             },
2300             "lineInterpolation": "linear",
2301             "lineWidth": 2,
2302             "pointSize": 5,
2303             "scaleDistribution": {
2304               "type": "linear"
2305             },
2306             "showPoints": "never",
2307             "spanNulls": false,
2308             "stacking": {
2309               "group": "A",
2310               "mode": "none"
2311             },
2312             "thresholdsStyle": {
2313               "mode": "off"
2314             }
2315           },
2316           "links": [],
2317           "mappings": [],
2318           "thresholds": {
2319             "mode": "absolute",
2320             "steps": [
2321               {
2322                 "color": "green",
2323                 "value": null
2324               },
2325               {
2326                 "color": "red",
2327                 "value": 80
2328               }
2329             ]
2330           },
2331           "unit": "cps"
2332         },
2333         "overrides": []
2334       },
2335       "gridPos": {
2336         "h": 8,
2337         "w": 8,
2338         "x": 0,
2339         "y": 36
2340       },
2341       "id": 10,
2342       "links": [],
2343       "options": {
2344         "legend": {
2345           "calcs": [
2346             "mean",
2347             "lastNotNull",
2348             "max"
2349           ],
2350           "displayMode": "table",
2351           "placement": "bottom",
2352           "showLegend": true
2353         },
2354         "tooltip": {
2355           "mode": "single",
2356           "sort": "none"
2357         }
2358       },
2359       "pluginVersion": "8.1.3",
2360       "targets": [
2361         {
2362           "datasource": {
2363             "type": "prometheus",
2364             "uid": "PBFA97CFB590B2093"
2365           },
2366           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2367           "format": "time_series",
2368           "interval": "",
2369           "intervalFactor": 2,
2370           "legendFormat": "{{instance}}",
2371           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2372           "refId": "A",
2373           "step": 4
2374         }
2375       ],
2376       "title": "Messages In Per Broker",
2377       "type": "timeseries"
2378     },
2379     {
2380       "datasource": {
2381         "type": "prometheus",
2382         "uid": "PBFA97CFB590B2093"
2383       },
2384       "fieldConfig": {
2385         "defaults": {
2386           "color": {
2387             "mode": "palette-classic"
2388           },
2389           "custom": {
2390             "axisCenteredZero": false,
2391             "axisColorMode": "text",
2392             "axisLabel": "Bytes/s",
2393             "axisPlacement": "auto",
2394             "barAlignment": 0,
2395             "drawStyle": "line",
2396             "fillOpacity": 10,
2397             "gradientMode": "none",
2398             "hideFrom": {
2399               "legend": false,
2400               "tooltip": false,
2401               "viz": false
2402             },
2403             "lineInterpolation": "linear",
2404             "lineWidth": 2,
2405             "pointSize": 5,
2406             "scaleDistribution": {
2407               "type": "linear"
2408             },
2409             "showPoints": "never",
2410             "spanNulls": false,
2411             "stacking": {
2412               "group": "A",
2413               "mode": "none"
2414             },
2415             "thresholdsStyle": {
2416               "mode": "off"
2417             }
2418           },
2419           "links": [],
2420           "mappings": [],
2421           "thresholds": {
2422             "mode": "absolute",
2423             "steps": [
2424               {
2425                 "color": "green",
2426                 "value": null
2427               },
2428               {
2429                 "color": "red",
2430                 "value": 80
2431               }
2432             ]
2433           },
2434           "unit": "Bps"
2435         },
2436         "overrides": []
2437       },
2438       "gridPos": {
2439         "h": 8,
2440         "w": 8,
2441         "x": 8,
2442         "y": 36
2443       },
2444       "id": 7,
2445       "links": [],
2446       "options": {
2447         "legend": {
2448           "calcs": [
2449             "mean",
2450             "lastNotNull",
2451             "max"
2452           ],
2453           "displayMode": "table",
2454           "placement": "bottom",
2455           "showLegend": true
2456         },
2457         "tooltip": {
2458           "mode": "single",
2459           "sort": "none"
2460         }
2461       },
2462       "pluginVersion": "8.1.3",
2463       "targets": [
2464         {
2465           "datasource": {
2466             "type": "prometheus",
2467             "uid": "PBFA97CFB590B2093"
2468           },
2469           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2470           "format": "time_series",
2471           "interval": "",
2472           "intervalFactor": 2,
2473           "legendFormat": "{{instance}}",
2474           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2475           "refId": "A",
2476           "step": 4
2477         }
2478       ],
2479       "title": "Bytes In Per Broker",
2480       "type": "timeseries"
2481     },
2482     {
2483       "datasource": {
2484         "type": "prometheus",
2485         "uid": "PBFA97CFB590B2093"
2486       },
2487       "fieldConfig": {
2488         "defaults": {
2489           "color": {
2490             "mode": "palette-classic"
2491           },
2492           "custom": {
2493             "axisCenteredZero": false,
2494             "axisColorMode": "text",
2495             "axisLabel": "",
2496             "axisPlacement": "auto",
2497             "barAlignment": 0,
2498             "drawStyle": "line",
2499             "fillOpacity": 10,
2500             "gradientMode": "none",
2501             "hideFrom": {
2502               "legend": false,
2503               "tooltip": false,
2504               "viz": false
2505             },
2506             "lineInterpolation": "linear",
2507             "lineWidth": 1,
2508             "pointSize": 5,
2509             "scaleDistribution": {
2510               "type": "linear"
2511             },
2512             "showPoints": "never",
2513             "spanNulls": true,
2514             "stacking": {
2515               "group": "A",
2516               "mode": "none"
2517             },
2518             "thresholdsStyle": {
2519               "mode": "off"
2520             }
2521           },
2522           "links": [],
2523           "mappings": [],
2524           "min": 0,
2525           "thresholds": {
2526             "mode": "absolute",
2527             "steps": [
2528               {
2529                 "color": "green",
2530                 "value": null
2531               },
2532               {
2533                 "color": "red",
2534                 "value": 80
2535               }
2536             ]
2537           },
2538           "unit": "binBps"
2539         },
2540         "overrides": []
2541       },
2542       "gridPos": {
2543         "h": 8,
2544         "w": 8,
2545         "x": 16,
2546         "y": 36
2547       },
2548       "id": 9,
2549       "links": [],
2550       "options": {
2551         "legend": {
2552           "calcs": [
2553             "mean",
2554             "lastNotNull",
2555             "max"
2556           ],
2557           "displayMode": "table",
2558           "placement": "bottom",
2559           "showLegend": true
2560         },
2561         "tooltip": {
2562           "mode": "single",
2563           "sort": "none"
2564         }
2565       },
2566       "pluginVersion": "8.1.3",
2567       "targets": [
2568         {
2569           "datasource": {
2570             "type": "prometheus",
2571             "uid": "PBFA97CFB590B2093"
2572           },
2573           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2574           "format": "time_series",
2575           "interval": "",
2576           "intervalFactor": 1,
2577           "legendFormat": "{{instance}}",
2578           "refId": "A"
2579         }
2580       ],
2581       "title": "Bytes Out Per Broker",
2582       "type": "timeseries"
2583     },
2584     {
2585       "collapsed": false,
2586       "datasource": {
2587         "type": "datasource",
2588         "uid": "grafana"
2589       },
2590       "gridPos": {
2591         "h": 1,
2592         "w": 24,
2593         "x": 0,
2594         "y": 44
2595       },
2596       "id": 44,
2597       "panels": [],
2598       "targets": [
2599         {
2600           "datasource": {
2601             "type": "datasource",
2602             "uid": "grafana"
2603           },
2604           "refId": "A"
2605         }
2606       ],
2607       "title": "Thread utilization",
2608       "type": "row"
2609     },
2610     {
2611       "datasource": {
2612         "type": "prometheus",
2613         "uid": "PBFA97CFB590B2093"
2614       },
2615       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2616       "fieldConfig": {
2617         "defaults": {
2618           "color": {
2619             "mode": "palette-classic"
2620           },
2621           "custom": {
2622             "axisCenteredZero": false,
2623             "axisColorMode": "text",
2624             "axisLabel": "",
2625             "axisPlacement": "auto",
2626             "barAlignment": 0,
2627             "drawStyle": "line",
2628             "fillOpacity": 10,
2629             "gradientMode": "none",
2630             "hideFrom": {
2631               "legend": false,
2632               "tooltip": false,
2633               "viz": false
2634             },
2635             "lineInterpolation": "linear",
2636             "lineWidth": 1,
2637             "pointSize": 5,
2638             "scaleDistribution": {
2639               "type": "linear"
2640             },
2641             "showPoints": "never",
2642             "spanNulls": true,
2643             "stacking": {
2644               "group": "A",
2645               "mode": "none"
2646             },
2647             "thresholdsStyle": {
2648               "mode": "off"
2649             }
2650           },
2651           "links": [],
2652           "mappings": [],
2653           "thresholds": {
2654             "mode": "absolute",
2655             "steps": [
2656               {
2657                 "color": "green",
2658                 "value": null
2659               },
2660               {
2661                 "color": "red",
2662                 "value": 80
2663               }
2664             ]
2665           },
2666           "unit": "percentunit"
2667         },
2668         "overrides": []
2669       },
2670       "gridPos": {
2671         "h": 12,
2672         "w": 12,
2673         "x": 0,
2674         "y": 45
2675       },
2676       "id": 24,
2677       "options": {
2678         "legend": {
2679           "calcs": [
2680             "mean",
2681             "lastNotNull",
2682             "max"
2683           ],
2684           "displayMode": "table",
2685           "placement": "bottom",
2686           "showLegend": true
2687         },
2688         "tooltip": {
2689           "mode": "single",
2690           "sort": "none"
2691         }
2692       },
2693       "pluginVersion": "8.1.3",
2694       "targets": [
2695         {
2696           "datasource": {
2697             "type": "prometheus",
2698             "uid": "PBFA97CFB590B2093"
2699           },
2700           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2701           "interval": "",
2702           "legendFormat": "{{instance}}",
2703           "refId": "A"
2704         }
2705       ],
2706       "title": "Network Processor Avg Usage Percent",
2707       "type": "timeseries"
2708     },
2709     {
2710       "datasource": {
2711         "type": "prometheus",
2712         "uid": "PBFA97CFB590B2093"
2713       },
2714       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2715       "fieldConfig": {
2716         "defaults": {
2717           "color": {
2718             "mode": "palette-classic"
2719           },
2720           "custom": {
2721             "axisCenteredZero": false,
2722             "axisColorMode": "text",
2723             "axisLabel": "",
2724             "axisPlacement": "auto",
2725             "barAlignment": 0,
2726             "drawStyle": "line",
2727             "fillOpacity": 10,
2728             "gradientMode": "none",
2729             "hideFrom": {
2730               "legend": false,
2731               "tooltip": false,
2732               "viz": false
2733             },
2734             "lineInterpolation": "linear",
2735             "lineWidth": 1,
2736             "pointSize": 5,
2737             "scaleDistribution": {
2738               "type": "linear"
2739             },
2740             "showPoints": "never",
2741             "spanNulls": true,
2742             "stacking": {
2743               "group": "A",
2744               "mode": "none"
2745             },
2746             "thresholdsStyle": {
2747               "mode": "off"
2748             }
2749           },
2750           "links": [],
2751           "mappings": [],
2752           "min": 0,
2753           "thresholds": {
2754             "mode": "absolute",
2755             "steps": [
2756               {
2757                 "color": "green",
2758                 "value": null
2759               },
2760               {
2761                 "color": "red",
2762                 "value": 80
2763               }
2764             ]
2765           },
2766           "unit": "percentunit"
2767         },
2768         "overrides": []
2769       },
2770       "gridPos": {
2771         "h": 12,
2772         "w": 12,
2773         "x": 12,
2774         "y": 45
2775       },
2776       "id": 25,
2777       "options": {
2778         "legend": {
2779           "calcs": [
2780             "mean",
2781             "lastNotNull",
2782             "max"
2783           ],
2784           "displayMode": "table",
2785           "placement": "bottom",
2786           "showLegend": true
2787         },
2788         "tooltip": {
2789           "mode": "single",
2790           "sort": "none"
2791         }
2792       },
2793       "pluginVersion": "8.1.3",
2794       "targets": [
2795         {
2796           "datasource": {
2797             "type": "prometheus",
2798             "uid": "PBFA97CFB590B2093"
2799           },
2800           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2801           "interval": "",
2802           "legendFormat": "{{instance}}",
2803           "refId": "A"
2804         }
2805       ],
2806       "title": "Request Handler Avg Percent",
2807       "type": "timeseries"
2808     },
2809     {
2810       "collapsed": false,
2811       "datasource": {
2812         "type": "datasource",
2813         "uid": "grafana"
2814       },
2815       "gridPos": {
2816         "h": 1,
2817         "w": 24,
2818         "x": 0,
2819         "y": 57
2820       },
2821       "id": 82,
2822       "panels": [],
2823       "targets": [
2824         {
2825           "datasource": {
2826             "type": "datasource",
2827             "uid": "grafana"
2828           },
2829           "refId": "A"
2830         }
2831       ],
2832       "title": "Isr Shrinks / Expands",
2833       "type": "row"
2834     },
2835     {
2836       "datasource": {
2837         "type": "prometheus",
2838         "uid": "PBFA97CFB590B2093"
2839       },
2840       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2841       "fieldConfig": {
2842         "defaults": {
2843           "color": {
2844             "mode": "palette-classic"
2845           },
2846           "custom": {
2847             "axisCenteredZero": false,
2848             "axisColorMode": "text",
2849             "axisLabel": "",
2850             "axisPlacement": "auto",
2851             "barAlignment": 0,
2852             "drawStyle": "line",
2853             "fillOpacity": 10,
2854             "gradientMode": "none",
2855             "hideFrom": {
2856               "legend": false,
2857               "tooltip": false,
2858               "viz": false
2859             },
2860             "lineInterpolation": "linear",
2861             "lineWidth": 1,
2862             "pointSize": 5,
2863             "scaleDistribution": {
2864               "type": "linear"
2865             },
2866             "showPoints": "never",
2867             "spanNulls": true,
2868             "stacking": {
2869               "group": "A",
2870               "mode": "none"
2871             },
2872             "thresholdsStyle": {
2873               "mode": "off"
2874             }
2875           },
2876           "links": [],
2877           "mappings": [],
2878           "min": 0,
2879           "thresholds": {
2880             "mode": "absolute",
2881             "steps": [
2882               {
2883                 "color": "green",
2884                 "value": null
2885               },
2886               {
2887                 "color": "red",
2888                 "value": 80
2889               }
2890             ]
2891           },
2892           "unit": "ops"
2893         },
2894         "overrides": [
2895           {
2896             "__systemRef": "hideSeriesFrom",
2897             "matcher": {
2898               "id": "byNames",
2899               "options": {
2900                 "mode": "exclude",
2901                 "names": [
2902                   "kafka-2:80"
2903                 ],
2904                 "prefix": "All except:",
2905                 "readOnly": true
2906               }
2907             },
2908             "properties": [
2909               {
2910                 "id": "custom.hideFrom",
2911                 "value": {
2912                   "legend": false,
2913                   "tooltip": false,
2914                   "viz": true
2915                 }
2916               }
2917             ]
2918           }
2919         ]
2920       },
2921       "gridPos": {
2922         "h": 9,
2923         "w": 12,
2924         "x": 0,
2925         "y": 58
2926       },
2927       "id": 80,
2928       "options": {
2929         "legend": {
2930           "calcs": [
2931             "mean",
2932             "lastNotNull",
2933             "max"
2934           ],
2935           "displayMode": "table",
2936           "placement": "bottom",
2937           "showLegend": true
2938         },
2939         "tooltip": {
2940           "mode": "single",
2941           "sort": "none"
2942         }
2943       },
2944       "pluginVersion": "8.1.3",
2945       "targets": [
2946         {
2947           "datasource": {
2948             "type": "prometheus",
2949             "uid": "PBFA97CFB590B2093"
2950           },
2951           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2952           "interval": "",
2953           "legendFormat": "{{instance}}",
2954           "refId": "A"
2955         }
2956       ],
2957       "title": "IsrShrinks per Sec",
2958       "type": "timeseries"
2959     },
2960     {
2961       "datasource": {
2962         "type": "prometheus",
2963         "uid": "PBFA97CFB590B2093"
2964       },
2965       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2966       "fieldConfig": {
2967         "defaults": {
2968           "color": {
2969             "mode": "palette-classic"
2970           },
2971           "custom": {
2972             "axisCenteredZero": false,
2973             "axisColorMode": "text",
2974             "axisLabel": "",
2975             "axisPlacement": "auto",
2976             "barAlignment": 0,
2977             "drawStyle": "line",
2978             "fillOpacity": 10,
2979             "gradientMode": "none",
2980             "hideFrom": {
2981               "legend": false,
2982               "tooltip": false,
2983               "viz": false
2984             },
2985             "lineInterpolation": "linear",
2986             "lineWidth": 1,
2987             "pointSize": 5,
2988             "scaleDistribution": {
2989               "type": "linear"
2990             },
2991             "showPoints": "never",
2992             "spanNulls": true,
2993             "stacking": {
2994               "group": "A",
2995               "mode": "none"
2996             },
2997             "thresholdsStyle": {
2998               "mode": "off"
2999             }
3000           },
3001           "links": [],
3002           "mappings": [],
3003           "min": 0,
3004           "thresholds": {
3005             "mode": "absolute",
3006             "steps": [
3007               {
3008                 "color": "green",
3009                 "value": null
3010               },
3011               {
3012                 "color": "red",
3013                 "value": 80
3014               }
3015             ]
3016           },
3017           "unit": "ops"
3018         },
3019         "overrides": [
3020           {
3021             "__systemRef": "hideSeriesFrom",
3022             "matcher": {
3023               "id": "byNames",
3024               "options": {
3025                 "mode": "exclude",
3026                 "names": [
3027                   "kafka-2:80"
3028                 ],
3029                 "prefix": "All except:",
3030                 "readOnly": true
3031               }
3032             },
3033             "properties": [
3034               {
3035                 "id": "custom.hideFrom",
3036                 "value": {
3037                   "legend": false,
3038                   "tooltip": false,
3039                   "viz": true
3040                 }
3041               }
3042             ]
3043           }
3044         ]
3045       },
3046       "gridPos": {
3047         "h": 9,
3048         "w": 12,
3049         "x": 12,
3050         "y": 58
3051       },
3052       "id": 83,
3053       "options": {
3054         "legend": {
3055           "calcs": [
3056             "mean",
3057             "lastNotNull",
3058             "max"
3059           ],
3060           "displayMode": "table",
3061           "placement": "bottom",
3062           "showLegend": true
3063         },
3064         "tooltip": {
3065           "mode": "single",
3066           "sort": "none"
3067         }
3068       },
3069       "pluginVersion": "8.1.3",
3070       "targets": [
3071         {
3072           "datasource": {
3073             "type": "prometheus",
3074             "uid": "PBFA97CFB590B2093"
3075           },
3076           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3077           "hide": false,
3078           "interval": "",
3079           "legendFormat": "{{instance}}",
3080           "refId": "A"
3081         }
3082       ],
3083       "title": "IsrExpands per Sec",
3084       "type": "timeseries"
3085     },
3086     {
3087       "collapsed": false,
3088       "datasource": {
3089         "type": "datasource",
3090         "uid": "grafana"
3091       },
3092       "gridPos": {
3093         "h": 1,
3094         "w": 24,
3095         "x": 0,
3096         "y": 67
3097       },
3098       "id": 53,
3099       "panels": [],
3100       "targets": [
3101         {
3102           "datasource": {
3103             "type": "datasource",
3104             "uid": "grafana"
3105           },
3106           "refId": "A"
3107         }
3108       ],
3109       "title": "Logs size",
3110       "type": "row"
3111     },
3112     {
3113       "datasource": {
3114         "type": "prometheus",
3115         "uid": "PBFA97CFB590B2093"
3116       },
3117       "fieldConfig": {
3118         "defaults": {
3119           "color": {
3120             "mode": "palette-classic"
3121           },
3122           "custom": {
3123             "axisCenteredZero": false,
3124             "axisColorMode": "text",
3125             "axisLabel": "",
3126             "axisPlacement": "auto",
3127             "barAlignment": 0,
3128             "drawStyle": "line",
3129             "fillOpacity": 10,
3130             "gradientMode": "none",
3131             "hideFrom": {
3132               "legend": false,
3133               "tooltip": false,
3134               "viz": false
3135             },
3136             "lineInterpolation": "linear",
3137             "lineWidth": 1,
3138             "pointSize": 5,
3139             "scaleDistribution": {
3140               "type": "linear"
3141             },
3142             "showPoints": "never",
3143             "spanNulls": true,
3144             "stacking": {
3145               "group": "A",
3146               "mode": "none"
3147             },
3148             "thresholdsStyle": {
3149               "mode": "off"
3150             }
3151           },
3152           "links": [],
3153           "mappings": [],
3154           "thresholds": {
3155             "mode": "absolute",
3156             "steps": [
3157               {
3158                 "color": "green",
3159                 "value": null
3160               },
3161               {
3162                 "color": "red",
3163                 "value": 80
3164               }
3165             ]
3166           },
3167           "unit": "bytes"
3168         },
3169         "overrides": []
3170       },
3171       "gridPos": {
3172         "h": 9,
3173         "w": 12,
3174         "x": 0,
3175         "y": 68
3176       },
3177       "id": 55,
3178       "options": {
3179         "legend": {
3180           "calcs": [
3181             "mean",
3182             "lastNotNull",
3183             "max"
3184           ],
3185           "displayMode": "table",
3186           "placement": "bottom",
3187           "showLegend": true
3188         },
3189         "tooltip": {
3190           "mode": "single",
3191           "sort": "none"
3192         }
3193       },
3194       "pluginVersion": "8.1.3",
3195       "targets": [
3196         {
3197           "datasource": {
3198             "type": "prometheus",
3199             "uid": "PBFA97CFB590B2093"
3200           },
3201           "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3202           "legendFormat": "{{topic}}",
3203           "refId": "A"
3204         }
3205       ],
3206       "title": "Log size per Topic",
3207       "type": "timeseries"
3208     },
3209     {
3210       "datasource": {
3211         "type": "prometheus",
3212         "uid": "PBFA97CFB590B2093"
3213       },
3214       "fieldConfig": {
3215         "defaults": {
3216           "color": {
3217             "mode": "palette-classic"
3218           },
3219           "custom": {
3220             "axisCenteredZero": false,
3221             "axisColorMode": "text",
3222             "axisLabel": "",
3223             "axisPlacement": "auto",
3224             "barAlignment": 0,
3225             "drawStyle": "line",
3226             "fillOpacity": 10,
3227             "gradientMode": "none",
3228             "hideFrom": {
3229               "legend": false,
3230               "tooltip": false,
3231               "viz": false
3232             },
3233             "lineInterpolation": "linear",
3234             "lineWidth": 1,
3235             "pointSize": 5,
3236             "scaleDistribution": {
3237               "type": "linear"
3238             },
3239             "showPoints": "never",
3240             "spanNulls": true,
3241             "stacking": {
3242               "group": "A",
3243               "mode": "none"
3244             },
3245             "thresholdsStyle": {
3246               "mode": "off"
3247             }
3248           },
3249           "links": [],
3250           "mappings": [],
3251           "thresholds": {
3252             "mode": "absolute",
3253             "steps": [
3254               {
3255                 "color": "green",
3256                 "value": null
3257               },
3258               {
3259                 "color": "red",
3260                 "value": 80
3261               }
3262             ]
3263           },
3264           "unit": "bytes"
3265         },
3266         "overrides": []
3267       },
3268       "gridPos": {
3269         "h": 9,
3270         "w": 12,
3271         "x": 12,
3272         "y": 68
3273       },
3274       "id": 56,
3275       "options": {
3276         "legend": {
3277           "calcs": [
3278             "mean",
3279             "lastNotNull",
3280             "max"
3281           ],
3282           "displayMode": "table",
3283           "placement": "bottom",
3284           "showLegend": true
3285         },
3286         "tooltip": {
3287           "mode": "single",
3288           "sort": "none"
3289         }
3290       },
3291       "pluginVersion": "8.1.3",
3292       "targets": [
3293         {
3294           "datasource": {
3295             "type": "prometheus",
3296             "uid": "PBFA97CFB590B2093"
3297           },
3298           "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3299           "legendFormat": "{{instance}}",
3300           "refId": "A"
3301         }
3302       ],
3303       "title": "Log size per Broker",
3304       "type": "timeseries"
3305     },
3306     {
3307       "collapsed": false,
3308       "datasource": {
3309         "type": "datasource",
3310         "uid": "grafana"
3311       },
3312       "gridPos": {
3313         "h": 1,
3314         "w": 24,
3315         "x": 0,
3316         "y": 77
3317       },
3318       "id": 58,
3319       "panels": [],
3320       "targets": [
3321         {
3322           "datasource": {
3323             "type": "datasource",
3324             "uid": "grafana"
3325           },
3326           "refId": "A"
3327         }
3328       ],
3329       "title": "Producer Performance",
3330       "type": "row"
3331     },
3332     {
3333       "datasource": {
3334         "type": "prometheus",
3335         "uid": "PBFA97CFB590B2093"
3336       },
3337       "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3338       "fieldConfig": {
3339         "defaults": {
3340           "color": {
3341             "mode": "palette-classic"
3342           },
3343           "custom": {
3344             "axisCenteredZero": false,
3345             "axisColorMode": "text",
3346             "axisLabel": "",
3347             "axisPlacement": "auto",
3348             "barAlignment": 0,
3349             "drawStyle": "line",
3350             "fillOpacity": 10,
3351             "gradientMode": "none",
3352             "hideFrom": {
3353               "legend": false,
3354               "tooltip": false,
3355               "viz": false
3356             },
3357             "lineInterpolation": "linear",
3358             "lineWidth": 1,
3359             "pointSize": 5,
3360             "scaleDistribution": {
3361               "type": "linear"
3362             },
3363             "showPoints": "never",
3364             "spanNulls": true,
3365             "stacking": {
3366               "group": "A",
3367               "mode": "none"
3368             },
3369             "thresholdsStyle": {
3370               "mode": "off"
3371             }
3372           },
3373           "links": [],
3374           "mappings": [],
3375           "thresholds": {
3376             "mode": "absolute",
3377             "steps": [
3378               {
3379                 "color": "green",
3380                 "value": null
3381               },
3382               {
3383                 "color": "red",
3384                 "value": 80
3385               }
3386             ]
3387           },
3388           "unit": "ms"
3389         },
3390         "overrides": []
3391       },
3392       "gridPos": {
3393         "h": 9,
3394         "w": 12,
3395         "x": 0,
3396         "y": 78
3397       },
3398       "id": 60,
3399       "options": {
3400         "legend": {
3401           "calcs": [
3402             "mean",
3403             "lastNotNull",
3404             "max"
3405           ],
3406           "displayMode": "list",
3407           "placement": "bottom",
3408           "showLegend": true
3409         },
3410         "tooltip": {
3411           "mode": "single",
3412           "sort": "none"
3413         }
3414       },
3415       "pluginVersion": "8.1.3",
3416       "targets": [
3417         {
3418           "datasource": {
3419             "type": "prometheus",
3420             "uid": "PBFA97CFB590B2093"
3421           },
3422           "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3423           "hide": false,
3424           "legendFormat": "{{instance}} - {{quantile}}",
3425           "refId": "A"
3426         }
3427       ],
3428       "title": "Producer - RequestQueueTimeMs",
3429       "type": "timeseries"
3430     },
3431     {
3432       "datasource": {
3433         "type": "prometheus",
3434         "uid": "PBFA97CFB590B2093"
3435       },
3436       "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3437       "fieldConfig": {
3438         "defaults": {
3439           "color": {
3440             "mode": "palette-classic"
3441           },
3442           "custom": {
3443             "axisCenteredZero": false,
3444             "axisColorMode": "text",
3445             "axisLabel": "",
3446             "axisPlacement": "auto",
3447             "barAlignment": 0,
3448             "drawStyle": "line",
3449             "fillOpacity": 10,
3450             "gradientMode": "none",
3451             "hideFrom": {
3452               "legend": false,
3453               "tooltip": false,
3454               "viz": false
3455             },
3456             "lineInterpolation": "linear",
3457             "lineWidth": 1,
3458             "pointSize": 5,
3459             "scaleDistribution": {
3460               "type": "linear"
3461             },
3462             "showPoints": "never",
3463             "spanNulls": true,
3464             "stacking": {
3465               "group": "A",
3466               "mode": "none"
3467             },
3468             "thresholdsStyle": {
3469               "mode": "off"
3470             }
3471           },
3472           "links": [],
3473           "mappings": [],
3474           "thresholds": {
3475             "mode": "absolute",
3476             "steps": [
3477               {
3478                 "color": "green",
3479                 "value": null
3480               },
3481               {
3482                 "color": "red",
3483                 "value": 80
3484               }
3485             ]
3486           },
3487           "unit": "ms"
3488         },
3489         "overrides": []
3490       },
3491       "gridPos": {
3492         "h": 9,
3493         "w": 12,
3494         "x": 12,
3495         "y": 78
3496       },
3497       "id": 61,
3498       "options": {
3499         "legend": {
3500           "calcs": [
3501             "mean",
3502             "lastNotNull",
3503             "max"
3504           ],
3505           "displayMode": "list",
3506           "placement": "bottom",
3507           "showLegend": true
3508         },
3509         "tooltip": {
3510           "mode": "single",
3511           "sort": "none"
3512         }
3513       },
3514       "pluginVersion": "8.1.3",
3515       "targets": [
3516         {
3517           "datasource": {
3518             "type": "prometheus",
3519             "uid": "PBFA97CFB590B2093"
3520           },
3521           "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3522           "hide": false,
3523           "legendFormat": "{{instance}} - {{quantile}}",
3524           "refId": "A"
3525         }
3526       ],
3527       "title": "Producer - LocalTimeMs",
3528       "type": "timeseries"
3529     },
3530     {
3531       "datasource": {
3532         "type": "prometheus",
3533         "uid": "PBFA97CFB590B2093"
3534       },
3535       "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3536       "fieldConfig": {
3537         "defaults": {
3538           "color": {
3539             "mode": "palette-classic"
3540           },
3541           "custom": {
3542             "axisCenteredZero": false,
3543             "axisColorMode": "text",
3544             "axisLabel": "",
3545             "axisPlacement": "auto",
3546             "barAlignment": 0,
3547             "drawStyle": "line",
3548             "fillOpacity": 10,
3549             "gradientMode": "none",
3550             "hideFrom": {
3551               "legend": false,
3552               "tooltip": false,
3553               "viz": false
3554             },
3555             "lineInterpolation": "linear",
3556             "lineWidth": 1,
3557             "pointSize": 5,
3558             "scaleDistribution": {
3559               "type": "linear"
3560             },
3561             "showPoints": "never",
3562             "spanNulls": true,
3563             "stacking": {
3564               "group": "A",
3565               "mode": "none"
3566             },
3567             "thresholdsStyle": {
3568               "mode": "off"
3569             }
3570           },
3571           "links": [],
3572           "mappings": [],
3573           "thresholds": {
3574             "mode": "absolute",
3575             "steps": [
3576               {
3577                 "color": "green",
3578                 "value": null
3579               },
3580               {
3581                 "color": "red",
3582                 "value": 80
3583               }
3584             ]
3585           },
3586           "unit": "ms"
3587         },
3588         "overrides": []
3589       },
3590       "gridPos": {
3591         "h": 10,
3592         "w": 8,
3593         "x": 0,
3594         "y": 87
3595       },
3596       "id": 62,
3597       "options": {
3598         "legend": {
3599           "calcs": [
3600             "mean",
3601             "lastNotNull",
3602             "max"
3603           ],
3604           "displayMode": "table",
3605           "placement": "bottom",
3606           "showLegend": true
3607         },
3608         "tooltip": {
3609           "mode": "single",
3610           "sort": "none"
3611         }
3612       },
3613       "pluginVersion": "8.1.3",
3614       "targets": [
3615         {
3616           "datasource": {
3617             "type": "prometheus",
3618             "uid": "PBFA97CFB590B2093"
3619           },
3620           "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3621           "hide": false,
3622           "legendFormat": "{{instance}} - {{quantile}}",
3623           "refId": "A"
3624         }
3625       ],
3626       "title": "Producer - RemoteTimeMs",
3627       "type": "timeseries"
3628     },
3629     {
3630       "datasource": {
3631         "type": "prometheus",
3632         "uid": "PBFA97CFB590B2093"
3633       },
3634       "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3635       "fieldConfig": {
3636         "defaults": {
3637           "color": {
3638             "mode": "palette-classic"
3639           },
3640           "custom": {
3641             "axisCenteredZero": false,
3642             "axisColorMode": "text",
3643             "axisLabel": "",
3644             "axisPlacement": "auto",
3645             "barAlignment": 0,
3646             "drawStyle": "line",
3647             "fillOpacity": 10,
3648             "gradientMode": "none",
3649             "hideFrom": {
3650               "legend": false,
3651               "tooltip": false,
3652               "viz": false
3653             },
3654             "lineInterpolation": "linear",
3655             "lineWidth": 1,
3656             "pointSize": 5,
3657             "scaleDistribution": {
3658               "type": "linear"
3659             },
3660             "showPoints": "never",
3661             "spanNulls": true,
3662             "stacking": {
3663               "group": "A",
3664               "mode": "none"
3665             },
3666             "thresholdsStyle": {
3667               "mode": "off"
3668             }
3669           },
3670           "links": [],
3671           "mappings": [],
3672           "thresholds": {
3673             "mode": "absolute",
3674             "steps": [
3675               {
3676                 "color": "green",
3677                 "value": null
3678               },
3679               {
3680                 "color": "red",
3681                 "value": 80
3682               }
3683             ]
3684           },
3685           "unit": "ms"
3686         },
3687         "overrides": []
3688       },
3689       "gridPos": {
3690         "h": 10,
3691         "w": 8,
3692         "x": 8,
3693         "y": 87
3694       },
3695       "id": 63,
3696       "options": {
3697         "legend": {
3698           "calcs": [
3699             "mean",
3700             "lastNotNull",
3701             "max"
3702           ],
3703           "displayMode": "table",
3704           "placement": "bottom",
3705           "showLegend": true
3706         },
3707         "tooltip": {
3708           "mode": "single",
3709           "sort": "none"
3710         }
3711       },
3712       "pluginVersion": "8.1.3",
3713       "targets": [
3714         {
3715           "datasource": {
3716             "type": "prometheus",
3717             "uid": "PBFA97CFB590B2093"
3718           },
3719           "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3720           "hide": false,
3721           "legendFormat": "{{instance}} - {{quantile}}",
3722           "refId": "A"
3723         }
3724       ],
3725       "title": "Producer - ResponseQueueTimeMs",
3726       "type": "timeseries"
3727     },
3728     {
3729       "datasource": {
3730         "type": "prometheus",
3731         "uid": "PBFA97CFB590B2093"
3732       },
3733       "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3734       "fieldConfig": {
3735         "defaults": {
3736           "color": {
3737             "mode": "palette-classic"
3738           },
3739           "custom": {
3740             "axisCenteredZero": false,
3741             "axisColorMode": "text",
3742             "axisLabel": "",
3743             "axisPlacement": "auto",
3744             "barAlignment": 0,
3745             "drawStyle": "line",
3746             "fillOpacity": 10,
3747             "gradientMode": "none",
3748             "hideFrom": {
3749               "legend": false,
3750               "tooltip": false,
3751               "viz": false
3752             },
3753             "lineInterpolation": "linear",
3754             "lineWidth": 1,
3755             "pointSize": 5,
3756             "scaleDistribution": {
3757               "type": "linear"
3758             },
3759             "showPoints": "never",
3760             "spanNulls": true,
3761             "stacking": {
3762               "group": "A",
3763               "mode": "none"
3764             },
3765             "thresholdsStyle": {
3766               "mode": "off"
3767             }
3768           },
3769           "links": [],
3770           "mappings": [],
3771           "min": 0,
3772           "thresholds": {
3773             "mode": "absolute",
3774             "steps": [
3775               {
3776                 "color": "green",
3777                 "value": null
3778               },
3779               {
3780                 "color": "red",
3781                 "value": 80
3782               }
3783             ]
3784           },
3785           "unit": "ms"
3786         },
3787         "overrides": []
3788       },
3789       "gridPos": {
3790         "h": 10,
3791         "w": 8,
3792         "x": 16,
3793         "y": 87
3794       },
3795       "id": 64,
3796       "options": {
3797         "legend": {
3798           "calcs": [
3799             "mean",
3800             "lastNotNull",
3801             "max"
3802           ],
3803           "displayMode": "table",
3804           "placement": "bottom",
3805           "showLegend": true
3806         },
3807         "tooltip": {
3808           "mode": "single",
3809           "sort": "none"
3810         }
3811       },
3812       "pluginVersion": "8.1.3",
3813       "targets": [
3814         {
3815           "datasource": {
3816             "type": "prometheus",
3817             "uid": "PBFA97CFB590B2093"
3818           },
3819           "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3820           "hide": false,
3821           "legendFormat": "{{instance}} - {{quantile}}",
3822           "refId": "A"
3823         }
3824       ],
3825       "title": "Producer - ResponseSendTimeMs",
3826       "type": "timeseries"
3827     },
3828     {
3829       "collapsed": true,
3830       "datasource": {
3831         "type": "datasource",
3832         "uid": "grafana"
3833       },
3834       "gridPos": {
3835         "h": 1,
3836         "w": 24,
3837         "x": 0,
3838         "y": 97
3839       },
3840       "id": 68,
3841       "panels": [
3842         {
3843           "datasource": {
3844             "type": "prometheus",
3845             "uid": "PBFA97CFB590B2093"
3846           },
3847           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3848           "fieldConfig": {
3849             "defaults": {
3850               "color": {
3851                 "mode": "palette-classic"
3852               },
3853               "custom": {
3854                 "axisLabel": "",
3855                 "axisPlacement": "auto",
3856                 "barAlignment": 0,
3857                 "drawStyle": "line",
3858                 "fillOpacity": 10,
3859                 "gradientMode": "none",
3860                 "hideFrom": {
3861                   "legend": false,
3862                   "tooltip": false,
3863                   "viz": false
3864                 },
3865                 "lineInterpolation": "linear",
3866                 "lineWidth": 1,
3867                 "pointSize": 5,
3868                 "scaleDistribution": {
3869                   "type": "linear"
3870                 },
3871                 "showPoints": "never",
3872                 "spanNulls": true,
3873                 "stacking": {
3874                   "group": "A",
3875                   "mode": "none"
3876                 },
3877                 "thresholdsStyle": {
3878                   "mode": "off"
3879                 }
3880               },
3881               "links": [],
3882               "mappings": [],
3883               "min": 0,
3884               "thresholds": {
3885                 "mode": "absolute",
3886                 "steps": [
3887                   {
3888                     "color": "green"
3889                   },
3890                   {
3891                     "color": "red",
3892                     "value": 80
3893                   }
3894                 ]
3895               },
3896               "unit": "ms"
3897             },
3898             "overrides": []
3899           },
3900           "gridPos": {
3901             "h": 9,
3902             "w": 12,
3903             "x": 0,
3904             "y": 30
3905           },
3906           "id": 69,
3907           "options": {
3908             "legend": {
3909               "calcs": [
3910                 "mean",
3911                 "lastNotNull",
3912                 "max"
3913               ],
3914               "displayMode": "list",
3915               "placement": "bottom",
3916               "showLegend": true
3917             },
3918             "tooltip": {
3919               "mode": "single"
3920             }
3921           },
3922           "pluginVersion": "8.1.3",
3923           "targets": [
3924             {
3925               "datasource": {
3926                 "type": "prometheus",
3927                 "uid": "PBFA97CFB590B2093"
3928               },
3929               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3930               "hide": false,
3931               "legendFormat": "{{instance}} - {{quantile}}",
3932               "refId": "A"
3933             }
3934           ],
3935           "title": "Consumer - RequestQueueTimeMs",
3936           "type": "timeseries"
3937         },
3938         {
3939           "datasource": {
3940             "type": "prometheus",
3941             "uid": "PBFA97CFB590B2093"
3942           },
3943           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3944           "fieldConfig": {
3945             "defaults": {
3946               "color": {
3947                 "mode": "palette-classic"
3948               },
3949               "custom": {
3950                 "axisLabel": "",
3951                 "axisPlacement": "auto",
3952                 "barAlignment": 0,
3953                 "drawStyle": "line",
3954                 "fillOpacity": 10,
3955                 "gradientMode": "none",
3956                 "hideFrom": {
3957                   "legend": false,
3958                   "tooltip": false,
3959                   "viz": false
3960                 },
3961                 "lineInterpolation": "linear",
3962                 "lineWidth": 1,
3963                 "pointSize": 5,
3964                 "scaleDistribution": {
3965                   "type": "linear"
3966                 },
3967                 "showPoints": "never",
3968                 "spanNulls": true,
3969                 "stacking": {
3970                   "group": "A",
3971                   "mode": "none"
3972                 },
3973                 "thresholdsStyle": {
3974                   "mode": "off"
3975                 }
3976               },
3977               "links": [],
3978               "mappings": [],
3979               "thresholds": {
3980                 "mode": "absolute",
3981                 "steps": [
3982                   {
3983                     "color": "green"
3984                   },
3985                   {
3986                     "color": "red",
3987                     "value": 80
3988                   }
3989                 ]
3990               },
3991               "unit": "ms"
3992             },
3993             "overrides": []
3994           },
3995           "gridPos": {
3996             "h": 9,
3997             "w": 12,
3998             "x": 12,
3999             "y": 30
4000           },
4001           "id": 70,
4002           "options": {
4003             "legend": {
4004               "calcs": [
4005                 "mean",
4006                 "lastNotNull",
4007                 "max"
4008               ],
4009               "displayMode": "list",
4010               "placement": "bottom",
4011               "showLegend": true
4012             },
4013             "tooltip": {
4014               "mode": "single"
4015             }
4016           },
4017           "pluginVersion": "8.1.3",
4018           "targets": [
4019             {
4020               "datasource": {
4021                 "type": "prometheus",
4022                 "uid": "PBFA97CFB590B2093"
4023               },
4024               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4025               "hide": false,
4026               "legendFormat": "{{instance}} - {{quantile}}",
4027               "refId": "A"
4028             }
4029           ],
4030           "title": "Consumer - LocalTimeMs",
4031           "type": "timeseries"
4032         },
4033         {
4034           "datasource": {
4035             "type": "prometheus",
4036             "uid": "PBFA97CFB590B2093"
4037           },
4038           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4039           "fieldConfig": {
4040             "defaults": {
4041               "color": {
4042                 "mode": "palette-classic"
4043               },
4044               "custom": {
4045                 "axisLabel": "",
4046                 "axisPlacement": "auto",
4047                 "barAlignment": 0,
4048                 "drawStyle": "line",
4049                 "fillOpacity": 10,
4050                 "gradientMode": "none",
4051                 "hideFrom": {
4052                   "legend": false,
4053                   "tooltip": false,
4054                   "viz": false
4055                 },
4056                 "lineInterpolation": "linear",
4057                 "lineWidth": 1,
4058                 "pointSize": 5,
4059                 "scaleDistribution": {
4060                   "type": "linear"
4061                 },
4062                 "showPoints": "never",
4063                 "spanNulls": true,
4064                 "stacking": {
4065                   "group": "A",
4066                   "mode": "none"
4067                 },
4068                 "thresholdsStyle": {
4069                   "mode": "off"
4070                 }
4071               },
4072               "links": [],
4073               "mappings": [],
4074               "thresholds": {
4075                 "mode": "absolute",
4076                 "steps": [
4077                   {
4078                     "color": "green"
4079                   },
4080                   {
4081                     "color": "red",
4082                     "value": 80
4083                   }
4084                 ]
4085               },
4086               "unit": "ms"
4087             },
4088             "overrides": []
4089           },
4090           "gridPos": {
4091             "h": 9,
4092             "w": 8,
4093             "x": 0,
4094             "y": 39
4095           },
4096           "id": 71,
4097           "options": {
4098             "legend": {
4099               "calcs": [
4100                 "mean",
4101                 "lastNotNull",
4102                 "max"
4103               ],
4104               "displayMode": "table",
4105               "placement": "bottom",
4106               "showLegend": true
4107             },
4108             "tooltip": {
4109               "mode": "single"
4110             }
4111           },
4112           "pluginVersion": "8.1.3",
4113           "targets": [
4114             {
4115               "datasource": {
4116                 "type": "prometheus",
4117                 "uid": "PBFA97CFB590B2093"
4118               },
4119               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4120               "hide": false,
4121               "legendFormat": "{{instance}} - {{quantile}}",
4122               "refId": "A"
4123             }
4124           ],
4125           "title": "Consumer - RemoteTimeMs",
4126           "type": "timeseries"
4127         },
4128         {
4129           "datasource": {
4130             "type": "prometheus",
4131             "uid": "PBFA97CFB590B2093"
4132           },
4133           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4134           "fieldConfig": {
4135             "defaults": {
4136               "color": {
4137                 "mode": "palette-classic"
4138               },
4139               "custom": {
4140                 "axisLabel": "",
4141                 "axisPlacement": "auto",
4142                 "barAlignment": 0,
4143                 "drawStyle": "line",
4144                 "fillOpacity": 10,
4145                 "gradientMode": "none",
4146                 "hideFrom": {
4147                   "legend": false,
4148                   "tooltip": false,
4149                   "viz": false
4150                 },
4151                 "lineInterpolation": "linear",
4152                 "lineWidth": 1,
4153                 "pointSize": 5,
4154                 "scaleDistribution": {
4155                   "type": "linear"
4156                 },
4157                 "showPoints": "never",
4158                 "spanNulls": true,
4159                 "stacking": {
4160                   "group": "A",
4161                   "mode": "none"
4162                 },
4163                 "thresholdsStyle": {
4164                   "mode": "off"
4165                 }
4166               },
4167               "links": [],
4168               "mappings": [],
4169               "min": 0,
4170               "thresholds": {
4171                 "mode": "absolute",
4172                 "steps": [
4173                   {
4174                     "color": "green"
4175                   },
4176                   {
4177                     "color": "red",
4178                     "value": 80
4179                   }
4180                 ]
4181               },
4182               "unit": "ms"
4183             },
4184             "overrides": []
4185           },
4186           "gridPos": {
4187             "h": 9,
4188             "w": 8,
4189             "x": 8,
4190             "y": 39
4191           },
4192           "id": 72,
4193           "options": {
4194             "legend": {
4195               "calcs": [
4196                 "mean",
4197                 "lastNotNull",
4198                 "max"
4199               ],
4200               "displayMode": "table",
4201               "placement": "bottom",
4202               "showLegend": true
4203             },
4204             "tooltip": {
4205               "mode": "single"
4206             }
4207           },
4208           "pluginVersion": "8.1.3",
4209           "targets": [
4210             {
4211               "datasource": {
4212                 "type": "prometheus",
4213                 "uid": "PBFA97CFB590B2093"
4214               },
4215               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4216               "hide": false,
4217               "legendFormat": "{{instance}} - {{quantile}}",
4218               "refId": "A"
4219             }
4220           ],
4221           "title": "Consumer - ResponseQueueTimeMs",
4222           "type": "timeseries"
4223         },
4224         {
4225           "datasource": {
4226             "type": "prometheus",
4227             "uid": "PBFA97CFB590B2093"
4228           },
4229           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4230           "fieldConfig": {
4231             "defaults": {
4232               "color": {
4233                 "mode": "palette-classic"
4234               },
4235               "custom": {
4236                 "axisLabel": "",
4237                 "axisPlacement": "auto",
4238                 "barAlignment": 0,
4239                 "drawStyle": "line",
4240                 "fillOpacity": 10,
4241                 "gradientMode": "none",
4242                 "hideFrom": {
4243                   "legend": false,
4244                   "tooltip": false,
4245                   "viz": false
4246                 },
4247                 "lineInterpolation": "linear",
4248                 "lineWidth": 1,
4249                 "pointSize": 5,
4250                 "scaleDistribution": {
4251                   "type": "linear"
4252                 },
4253                 "showPoints": "never",
4254                 "spanNulls": true,
4255                 "stacking": {
4256                   "group": "A",
4257                   "mode": "none"
4258                 },
4259                 "thresholdsStyle": {
4260                   "mode": "off"
4261                 }
4262               },
4263               "links": [],
4264               "mappings": [],
4265               "min": 0,
4266               "thresholds": {
4267                 "mode": "absolute",
4268                 "steps": [
4269                   {
4270                     "color": "green"
4271                   },
4272                   {
4273                     "color": "red",
4274                     "value": 80
4275                   }
4276                 ]
4277               },
4278               "unit": "ms"
4279             },
4280             "overrides": []
4281           },
4282           "gridPos": {
4283             "h": 9,
4284             "w": 8,
4285             "x": 16,
4286             "y": 39
4287           },
4288           "id": 73,
4289           "options": {
4290             "legend": {
4291               "calcs": [
4292                 "mean",
4293                 "lastNotNull",
4294                 "max"
4295               ],
4296               "displayMode": "table",
4297               "placement": "bottom",
4298               "showLegend": true
4299             },
4300             "tooltip": {
4301               "mode": "single"
4302             }
4303           },
4304           "pluginVersion": "8.1.3",
4305           "targets": [
4306             {
4307               "datasource": {
4308                 "type": "prometheus",
4309                 "uid": "PBFA97CFB590B2093"
4310               },
4311               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4312               "hide": false,
4313               "legendFormat": "{{instance}} - {{quantile}}",
4314               "refId": "A"
4315             }
4316           ],
4317           "title": "Consumer - ResponseSendTimeMs",
4318           "type": "timeseries"
4319         }
4320       ],
4321       "targets": [
4322         {
4323           "datasource": {
4324             "type": "datasource",
4325             "uid": "grafana"
4326           },
4327           "refId": "A"
4328         }
4329       ],
4330       "title": "Consumer Performance",
4331       "type": "row"
4332     },
4333     {
4334       "collapsed": false,
4335       "datasource": {
4336         "type": "datasource",
4337         "uid": "grafana"
4338       },
4339       "gridPos": {
4340         "h": 1,
4341         "w": 24,
4342         "x": 0,
4343         "y": 98
4344       },
4345       "id": 66,
4346       "panels": [],
4347       "targets": [
4348         {
4349           "datasource": {
4350             "type": "datasource",
4351             "uid": "grafana"
4352           },
4353           "refId": "A"
4354         }
4355       ],
4356       "title": "Fetch Follower Performance",
4357       "type": "row"
4358     },
4359     {
4360       "datasource": {
4361         "type": "prometheus",
4362         "uid": "PBFA97CFB590B2093"
4363       },
4364       "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4365       "fieldConfig": {
4366         "defaults": {
4367           "color": {
4368             "mode": "palette-classic"
4369           },
4370           "custom": {
4371             "axisCenteredZero": false,
4372             "axisColorMode": "text",
4373             "axisLabel": "",
4374             "axisPlacement": "auto",
4375             "barAlignment": 0,
4376             "drawStyle": "line",
4377             "fillOpacity": 10,
4378             "gradientMode": "none",
4379             "hideFrom": {
4380               "legend": false,
4381               "tooltip": false,
4382               "viz": false
4383             },
4384             "lineInterpolation": "linear",
4385             "lineWidth": 1,
4386             "pointSize": 5,
4387             "scaleDistribution": {
4388               "type": "linear"
4389             },
4390             "showPoints": "never",
4391             "spanNulls": true,
4392             "stacking": {
4393               "group": "A",
4394               "mode": "none"
4395             },
4396             "thresholdsStyle": {
4397               "mode": "off"
4398             }
4399           },
4400           "links": [],
4401           "mappings": [],
4402           "min": 0,
4403           "thresholds": {
4404             "mode": "absolute",
4405             "steps": [
4406               {
4407                 "color": "green",
4408                 "value": null
4409               },
4410               {
4411                 "color": "red",
4412                 "value": 80
4413               }
4414             ]
4415           },
4416           "unit": "ms"
4417         },
4418         "overrides": []
4419       },
4420       "gridPos": {
4421         "h": 9,
4422         "w": 12,
4423         "x": 0,
4424         "y": 99
4425       },
4426       "id": 74,
4427       "options": {
4428         "legend": {
4429           "calcs": [
4430             "mean",
4431             "lastNotNull",
4432             "max"
4433           ],
4434           "displayMode": "list",
4435           "placement": "bottom",
4436           "showLegend": true
4437         },
4438         "tooltip": {
4439           "mode": "single",
4440           "sort": "none"
4441         }
4442       },
4443       "pluginVersion": "8.1.3",
4444       "targets": [
4445         {
4446           "datasource": {
4447             "type": "prometheus",
4448             "uid": "PBFA97CFB590B2093"
4449           },
4450           "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4451           "hide": false,
4452           "legendFormat": "{{instance}} - {{quantile}}",
4453           "refId": "A"
4454         }
4455       ],
4456       "title": "FetchFollower - RequestQueueTimeMs",
4457       "type": "timeseries"
4458     },
4459     {
4460       "datasource": {
4461         "type": "prometheus",
4462         "uid": "PBFA97CFB590B2093"
4463       },
4464       "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4465       "fieldConfig": {
4466         "defaults": {
4467           "color": {
4468             "mode": "palette-classic"
4469           },
4470           "custom": {
4471             "axisCenteredZero": false,
4472             "axisColorMode": "text",
4473             "axisLabel": "",
4474             "axisPlacement": "auto",
4475             "barAlignment": 0,
4476             "drawStyle": "line",
4477             "fillOpacity": 10,
4478             "gradientMode": "none",
4479             "hideFrom": {
4480               "legend": false,
4481               "tooltip": false,
4482               "viz": false
4483             },
4484             "lineInterpolation": "linear",
4485             "lineWidth": 1,
4486             "pointSize": 5,
4487             "scaleDistribution": {
4488               "type": "linear"
4489             },
4490             "showPoints": "never",
4491             "spanNulls": true,
4492             "stacking": {
4493               "group": "A",
4494               "mode": "none"
4495             },
4496             "thresholdsStyle": {
4497               "mode": "off"
4498             }
4499           },
4500           "links": [],
4501           "mappings": [],
4502           "min": 0,
4503           "thresholds": {
4504             "mode": "absolute",
4505             "steps": [
4506               {
4507                 "color": "green",
4508                 "value": null
4509               },
4510               {
4511                 "color": "red",
4512                 "value": 80
4513               }
4514             ]
4515           },
4516           "unit": "ms"
4517         },
4518         "overrides": []
4519       },
4520       "gridPos": {
4521         "h": 9,
4522         "w": 12,
4523         "x": 12,
4524         "y": 99
4525       },
4526       "id": 75,
4527       "options": {
4528         "legend": {
4529           "calcs": [
4530             "mean",
4531             "lastNotNull",
4532             "max"
4533           ],
4534           "displayMode": "list",
4535           "placement": "bottom",
4536           "showLegend": true
4537         },
4538         "tooltip": {
4539           "mode": "single",
4540           "sort": "none"
4541         }
4542       },
4543       "pluginVersion": "8.1.3",
4544       "targets": [
4545         {
4546           "datasource": {
4547             "type": "prometheus",
4548             "uid": "PBFA97CFB590B2093"
4549           },
4550           "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4551           "hide": false,
4552           "legendFormat": "{{instance}} - {{quantile}}",
4553           "refId": "A"
4554         }
4555       ],
4556       "title": "FetchFollower - LocalTimeMs",
4557       "type": "timeseries"
4558     },
4559     {
4560       "datasource": {
4561         "type": "prometheus",
4562         "uid": "PBFA97CFB590B2093"
4563       },
4564       "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4565       "fieldConfig": {
4566         "defaults": {
4567           "color": {
4568             "mode": "palette-classic"
4569           },
4570           "custom": {
4571             "axisCenteredZero": false,
4572             "axisColorMode": "text",
4573             "axisLabel": "",
4574             "axisPlacement": "auto",
4575             "barAlignment": 0,
4576             "drawStyle": "line",
4577             "fillOpacity": 10,
4578             "gradientMode": "none",
4579             "hideFrom": {
4580               "legend": false,
4581               "tooltip": false,
4582               "viz": false
4583             },
4584             "lineInterpolation": "linear",
4585             "lineWidth": 1,
4586             "pointSize": 5,
4587             "scaleDistribution": {
4588               "type": "linear"
4589             },
4590             "showPoints": "never",
4591             "spanNulls": true,
4592             "stacking": {
4593               "group": "A",
4594               "mode": "none"
4595             },
4596             "thresholdsStyle": {
4597               "mode": "off"
4598             }
4599           },
4600           "links": [],
4601           "mappings": [],
4602           "thresholds": {
4603             "mode": "absolute",
4604             "steps": [
4605               {
4606                 "color": "green",
4607                 "value": null
4608               },
4609               {
4610                 "color": "red",
4611                 "value": 80
4612               }
4613             ]
4614           },
4615           "unit": "ms"
4616         },
4617         "overrides": []
4618       },
4619       "gridPos": {
4620         "h": 9,
4621         "w": 8,
4622         "x": 0,
4623         "y": 108
4624       },
4625       "id": 76,
4626       "options": {
4627         "legend": {
4628           "calcs": [
4629             "mean",
4630             "lastNotNull",
4631             "max"
4632           ],
4633           "displayMode": "table",
4634           "placement": "bottom",
4635           "showLegend": true
4636         },
4637         "tooltip": {
4638           "mode": "single",
4639           "sort": "none"
4640         }
4641       },
4642       "pluginVersion": "8.1.3",
4643       "targets": [
4644         {
4645           "datasource": {
4646             "type": "prometheus",
4647             "uid": "PBFA97CFB590B2093"
4648           },
4649           "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4650           "hide": false,
4651           "legendFormat": "{{instance}} - {{quantile}}",
4652           "refId": "A"
4653         }
4654       ],
4655       "title": "FetchFollower - RemoteTimeMs",
4656       "type": "timeseries"
4657     },
4658     {
4659       "datasource": {
4660         "type": "prometheus",
4661         "uid": "PBFA97CFB590B2093"
4662       },
4663       "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4664       "fieldConfig": {
4665         "defaults": {
4666           "color": {
4667             "mode": "palette-classic"
4668           },
4669           "custom": {
4670             "axisCenteredZero": false,
4671             "axisColorMode": "text",
4672             "axisLabel": "",
4673             "axisPlacement": "auto",
4674             "barAlignment": 0,
4675             "drawStyle": "line",
4676             "fillOpacity": 10,
4677             "gradientMode": "none",
4678             "hideFrom": {
4679               "legend": false,
4680               "tooltip": false,
4681               "viz": false
4682             },
4683             "lineInterpolation": "linear",
4684             "lineWidth": 1,
4685             "pointSize": 5,
4686             "scaleDistribution": {
4687               "type": "linear"
4688             },
4689             "showPoints": "never",
4690             "spanNulls": true,
4691             "stacking": {
4692               "group": "A",
4693               "mode": "none"
4694             },
4695             "thresholdsStyle": {
4696               "mode": "off"
4697             }
4698           },
4699           "links": [],
4700           "mappings": [],
4701           "thresholds": {
4702             "mode": "absolute",
4703             "steps": [
4704               {
4705                 "color": "green",
4706                 "value": null
4707               },
4708               {
4709                 "color": "red",
4710                 "value": 80
4711               }
4712             ]
4713           },
4714           "unit": "ms"
4715         },
4716         "overrides": []
4717       },
4718       "gridPos": {
4719         "h": 9,
4720         "w": 8,
4721         "x": 8,
4722         "y": 108
4723       },
4724       "id": 77,
4725       "options": {
4726         "legend": {
4727           "calcs": [
4728             "mean",
4729             "lastNotNull",
4730             "max"
4731           ],
4732           "displayMode": "table",
4733           "placement": "bottom",
4734           "showLegend": true
4735         },
4736         "tooltip": {
4737           "mode": "single",
4738           "sort": "none"
4739         }
4740       },
4741       "pluginVersion": "8.1.3",
4742       "targets": [
4743         {
4744           "datasource": {
4745             "type": "prometheus",
4746             "uid": "PBFA97CFB590B2093"
4747           },
4748           "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4749           "hide": false,
4750           "legendFormat": "{{instance}} - {{quantile}}",
4751           "refId": "A"
4752         }
4753       ],
4754       "title": "FetchFollower - ResponseQueueTimeMs",
4755       "type": "timeseries"
4756     },
4757     {
4758       "datasource": {
4759         "type": "prometheus",
4760         "uid": "PBFA97CFB590B2093"
4761       },
4762       "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4763       "fieldConfig": {
4764         "defaults": {
4765           "color": {
4766             "mode": "palette-classic"
4767           },
4768           "custom": {
4769             "axisCenteredZero": false,
4770             "axisColorMode": "text",
4771             "axisLabel": "",
4772             "axisPlacement": "auto",
4773             "barAlignment": 0,
4774             "drawStyle": "line",
4775             "fillOpacity": 10,
4776             "gradientMode": "none",
4777             "hideFrom": {
4778               "legend": false,
4779               "tooltip": false,
4780               "viz": false
4781             },
4782             "lineInterpolation": "linear",
4783             "lineWidth": 1,
4784             "pointSize": 5,
4785             "scaleDistribution": {
4786               "type": "linear"
4787             },
4788             "showPoints": "never",
4789             "spanNulls": true,
4790             "stacking": {
4791               "group": "A",
4792               "mode": "none"
4793             },
4794             "thresholdsStyle": {
4795               "mode": "off"
4796             }
4797           },
4798           "links": [],
4799           "mappings": [],
4800           "min": 0,
4801           "thresholds": {
4802             "mode": "absolute",
4803             "steps": [
4804               {
4805                 "color": "green",
4806                 "value": null
4807               },
4808               {
4809                 "color": "red",
4810                 "value": 80
4811               }
4812             ]
4813           },
4814           "unit": "ms"
4815         },
4816         "overrides": []
4817       },
4818       "gridPos": {
4819         "h": 9,
4820         "w": 8,
4821         "x": 16,
4822         "y": 108
4823       },
4824       "id": 78,
4825       "options": {
4826         "legend": {
4827           "calcs": [
4828             "mean",
4829             "lastNotNull",
4830             "max"
4831           ],
4832           "displayMode": "table",
4833           "placement": "bottom",
4834           "showLegend": true
4835         },
4836         "tooltip": {
4837           "mode": "single",
4838           "sort": "none"
4839         }
4840       },
4841       "pluginVersion": "8.1.3",
4842       "targets": [
4843         {
4844           "datasource": {
4845             "type": "prometheus",
4846             "uid": "PBFA97CFB590B2093"
4847           },
4848           "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4849           "hide": false,
4850           "legendFormat": "{{instance}} - {{quantile}}",
4851           "refId": "A"
4852         }
4853       ],
4854       "title": "FetchFollower - ResponseSendTimeMs",
4855       "type": "timeseries"
4856     },
4857     {
4858       "collapsed": false,
4859       "datasource": {
4860         "type": "datasource",
4861         "uid": "grafana"
4862       },
4863       "gridPos": {
4864         "h": 1,
4865         "w": 24,
4866         "x": 0,
4867         "y": 117
4868       },
4869       "id": 102,
4870       "panels": [],
4871       "targets": [
4872         {
4873           "datasource": {
4874             "type": "datasource",
4875             "uid": "grafana"
4876           },
4877           "refId": "A"
4878         }
4879       ],
4880       "title": "Connections",
4881       "type": "row"
4882     },
4883     {
4884       "datasource": {
4885         "type": "prometheus",
4886         "uid": "PBFA97CFB590B2093"
4887       },
4888       "fieldConfig": {
4889         "defaults": {
4890           "color": {
4891             "mode": "palette-classic"
4892           },
4893           "custom": {
4894             "axisCenteredZero": false,
4895             "axisColorMode": "text",
4896             "axisLabel": "",
4897             "axisPlacement": "auto",
4898             "barAlignment": 0,
4899             "drawStyle": "line",
4900             "fillOpacity": 10,
4901             "gradientMode": "none",
4902             "hideFrom": {
4903               "legend": false,
4904               "tooltip": false,
4905               "viz": false
4906             },
4907             "lineInterpolation": "linear",
4908             "lineWidth": 1,
4909             "pointSize": 5,
4910             "scaleDistribution": {
4911               "type": "linear"
4912             },
4913             "showPoints": "never",
4914             "spanNulls": true,
4915             "stacking": {
4916               "group": "A",
4917               "mode": "none"
4918             },
4919             "thresholdsStyle": {
4920               "mode": "off"
4921             }
4922           },
4923           "links": [],
4924           "mappings": [],
4925           "thresholds": {
4926             "mode": "absolute",
4927             "steps": [
4928               {
4929                 "color": "green",
4930                 "value": null
4931               },
4932               {
4933                 "color": "red",
4934                 "value": 80
4935               }
4936             ]
4937           },
4938           "unit": "short"
4939         },
4940         "overrides": []
4941       },
4942       "gridPos": {
4943         "h": 8,
4944         "w": 12,
4945         "x": 0,
4946         "y": 118
4947       },
4948       "id": 98,
4949       "options": {
4950         "legend": {
4951           "calcs": [
4952             "mean",
4953             "lastNotNull",
4954             "max"
4955           ],
4956           "displayMode": "table",
4957           "placement": "bottom",
4958           "showLegend": true
4959         },
4960         "tooltip": {
4961           "mode": "single",
4962           "sort": "none"
4963         }
4964       },
4965       "pluginVersion": "8.1.3",
4966       "targets": [
4967         {
4968           "datasource": {
4969             "type": "prometheus",
4970             "uid": "PBFA97CFB590B2093"
4971           },
4972           "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4973           "interval": "",
4974           "legendFormat": "{{listener}}",
4975           "refId": "A"
4976         }
4977       ],
4978       "title": "Connections count per listener",
4979       "type": "timeseries"
4980     },
4981     {
4982       "datasource": {
4983         "type": "prometheus",
4984         "uid": "PBFA97CFB590B2093"
4985       },
4986       "fieldConfig": {
4987         "defaults": {
4988           "color": {
4989             "mode": "palette-classic"
4990           },
4991           "custom": {
4992             "axisCenteredZero": false,
4993             "axisColorMode": "text",
4994             "axisLabel": "",
4995             "axisPlacement": "auto",
4996             "barAlignment": 0,
4997             "drawStyle": "line",
4998             "fillOpacity": 10,
4999             "gradientMode": "none",
5000             "hideFrom": {
5001               "legend": false,
5002               "tooltip": false,
5003               "viz": false
5004             },
5005             "lineInterpolation": "linear",
5006             "lineWidth": 1,
5007             "pointSize": 5,
5008             "scaleDistribution": {
5009               "type": "linear"
5010             },
5011             "showPoints": "never",
5012             "spanNulls": true,
5013             "stacking": {
5014               "group": "A",
5015               "mode": "none"
5016             },
5017             "thresholdsStyle": {
5018               "mode": "off"
5019             }
5020           },
5021           "links": [],
5022           "mappings": [],
5023           "thresholds": {
5024             "mode": "absolute",
5025             "steps": [
5026               {
5027                 "color": "green",
5028                 "value": null
5029               },
5030               {
5031                 "color": "red",
5032                 "value": 80
5033               }
5034             ]
5035           },
5036           "unit": "short"
5037         },
5038         "overrides": []
5039       },
5040       "gridPos": {
5041         "h": 8,
5042         "w": 12,
5043         "x": 12,
5044         "y": 118
5045       },
5046       "id": 100,
5047       "options": {
5048         "legend": {
5049           "calcs": [
5050             "mean",
5051             "lastNotNull",
5052             "max"
5053           ],
5054           "displayMode": "table",
5055           "placement": "bottom",
5056           "showLegend": true
5057         },
5058         "tooltip": {
5059           "mode": "single",
5060           "sort": "none"
5061         }
5062       },
5063       "pluginVersion": "8.1.3",
5064       "targets": [
5065         {
5066           "datasource": {
5067             "type": "prometheus",
5068             "uid": "PBFA97CFB590B2093"
5069           },
5070           "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5071           "interval": "",
5072           "legendFormat": "{{instance}}",
5073           "refId": "A"
5074         }
5075       ],
5076       "title": "Connections count per broker",
5077       "type": "timeseries"
5078     },
5079     {
5080       "datasource": {
5081         "type": "prometheus",
5082         "uid": "PBFA97CFB590B2093"
5083       },
5084       "fieldConfig": {
5085         "defaults": {
5086           "color": {
5087             "mode": "palette-classic"
5088           },
5089           "custom": {
5090             "axisCenteredZero": false,
5091             "axisColorMode": "text",
5092             "axisLabel": "",
5093             "axisPlacement": "auto",
5094             "barAlignment": 0,
5095             "drawStyle": "line",
5096             "fillOpacity": 10,
5097             "gradientMode": "none",
5098             "hideFrom": {
5099               "legend": false,
5100               "tooltip": false,
5101               "viz": false
5102             },
5103             "lineInterpolation": "linear",
5104             "lineWidth": 1,
5105             "pointSize": 5,
5106             "scaleDistribution": {
5107               "type": "linear"
5108             },
5109             "showPoints": "never",
5110             "spanNulls": true,
5111             "stacking": {
5112               "group": "A",
5113               "mode": "none"
5114             },
5115             "thresholdsStyle": {
5116               "mode": "off"
5117             }
5118           },
5119           "links": [],
5120           "mappings": [],
5121           "thresholds": {
5122             "mode": "absolute",
5123             "steps": [
5124               {
5125                 "color": "green",
5126                 "value": null
5127               },
5128               {
5129                 "color": "red",
5130                 "value": 80
5131               }
5132             ]
5133           },
5134           "unit": "short"
5135         },
5136         "overrides": []
5137       },
5138       "gridPos": {
5139         "h": 8,
5140         "w": 12,
5141         "x": 0,
5142         "y": 126
5143       },
5144       "id": 104,
5145       "options": {
5146         "legend": {
5147           "calcs": [
5148             "mean",
5149             "lastNotNull",
5150             "max"
5151           ],
5152           "displayMode": "table",
5153           "placement": "bottom",
5154           "showLegend": true
5155         },
5156         "tooltip": {
5157           "mode": "single",
5158           "sort": "none"
5159         }
5160       },
5161       "pluginVersion": "8.1.3",
5162       "targets": [
5163         {
5164           "datasource": {
5165             "type": "prometheus",
5166             "uid": "PBFA97CFB590B2093"
5167           },
5168           "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5169           "interval": "",
5170           "legendFormat": "{{listener}}",
5171           "refId": "A"
5172         }
5173       ],
5174       "title": "Connections creation rate per listener",
5175       "type": "timeseries"
5176     },
5177     {
5178       "datasource": {
5179         "type": "prometheus",
5180         "uid": "PBFA97CFB590B2093"
5181       },
5182       "fieldConfig": {
5183         "defaults": {
5184           "color": {
5185             "mode": "palette-classic"
5186           },
5187           "custom": {
5188             "axisCenteredZero": false,
5189             "axisColorMode": "text",
5190             "axisLabel": "",
5191             "axisPlacement": "auto",
5192             "barAlignment": 0,
5193             "drawStyle": "line",
5194             "fillOpacity": 10,
5195             "gradientMode": "none",
5196             "hideFrom": {
5197               "legend": false,
5198               "tooltip": false,
5199               "viz": false
5200             },
5201             "lineInterpolation": "linear",
5202             "lineWidth": 1,
5203             "pointSize": 5,
5204             "scaleDistribution": {
5205               "type": "linear"
5206             },
5207             "showPoints": "never",
5208             "spanNulls": true,
5209             "stacking": {
5210               "group": "A",
5211               "mode": "none"
5212             },
5213             "thresholdsStyle": {
5214               "mode": "off"
5215             }
5216           },
5217           "links": [],
5218           "mappings": [],
5219           "thresholds": {
5220             "mode": "absolute",
5221             "steps": [
5222               {
5223                 "color": "green",
5224                 "value": null
5225               },
5226               {
5227                 "color": "red",
5228                 "value": 80
5229               }
5230             ]
5231           },
5232           "unit": "short"
5233         },
5234         "overrides": []
5235       },
5236       "gridPos": {
5237         "h": 8,
5238         "w": 12,
5239         "x": 12,
5240         "y": 126
5241       },
5242       "id": 106,
5243       "options": {
5244         "legend": {
5245           "calcs": [
5246             "mean",
5247             "lastNotNull",
5248             "max"
5249           ],
5250           "displayMode": "table",
5251           "placement": "bottom",
5252           "showLegend": true
5253         },
5254         "tooltip": {
5255           "mode": "single",
5256           "sort": "none"
5257         }
5258       },
5259       "pluginVersion": "8.1.3",
5260       "targets": [
5261         {
5262           "datasource": {
5263             "type": "prometheus",
5264             "uid": "PBFA97CFB590B2093"
5265           },
5266           "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5267           "interval": "",
5268           "legendFormat": "{{instance}}",
5269           "refId": "A"
5270         }
5271       ],
5272       "title": "Connections creation rate per instance",
5273       "type": "timeseries"
5274     },
5275     {
5276       "datasource": {
5277         "type": "prometheus",
5278         "uid": "PBFA97CFB590B2093"
5279       },
5280       "fieldConfig": {
5281         "defaults": {
5282           "color": {
5283             "mode": "palette-classic"
5284           },
5285           "custom": {
5286             "axisCenteredZero": false,
5287             "axisColorMode": "text",
5288             "axisLabel": "",
5289             "axisPlacement": "auto",
5290             "barAlignment": 0,
5291             "drawStyle": "line",
5292             "fillOpacity": 10,
5293             "gradientMode": "none",
5294             "hideFrom": {
5295               "legend": false,
5296               "tooltip": false,
5297               "viz": false
5298             },
5299             "lineInterpolation": "linear",
5300             "lineWidth": 1,
5301             "pointSize": 5,
5302             "scaleDistribution": {
5303               "type": "linear"
5304             },
5305             "showPoints": "never",
5306             "spanNulls": true,
5307             "stacking": {
5308               "group": "A",
5309               "mode": "none"
5310             },
5311             "thresholdsStyle": {
5312               "mode": "off"
5313             }
5314           },
5315           "links": [],
5316           "mappings": [],
5317           "thresholds": {
5318             "mode": "absolute",
5319             "steps": [
5320               {
5321                 "color": "green",
5322                 "value": null
5323               },
5324               {
5325                 "color": "red",
5326                 "value": 80
5327               }
5328             ]
5329           },
5330           "unit": "short"
5331         },
5332         "overrides": []
5333       },
5334       "gridPos": {
5335         "h": 8,
5336         "w": 12,
5337         "x": 0,
5338         "y": 134
5339       },
5340       "id": 108,
5341       "options": {
5342         "legend": {
5343           "calcs": [
5344             "mean",
5345             "lastNotNull",
5346             "max"
5347           ],
5348           "displayMode": "table",
5349           "placement": "bottom",
5350           "showLegend": true
5351         },
5352         "tooltip": {
5353           "mode": "single",
5354           "sort": "none"
5355         }
5356       },
5357       "pluginVersion": "8.1.3",
5358       "targets": [
5359         {
5360           "datasource": {
5361             "type": "prometheus",
5362             "uid": "PBFA97CFB590B2093"
5363           },
5364           "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5365           "interval": "",
5366           "legendFormat": "{{listener}}",
5367           "refId": "A"
5368         }
5369       ],
5370       "title": "Connections close rate per listener",
5371       "type": "timeseries"
5372     },
5373     {
5374       "datasource": {
5375         "type": "prometheus",
5376         "uid": "PBFA97CFB590B2093"
5377       },
5378       "fieldConfig": {
5379         "defaults": {
5380           "color": {
5381             "mode": "palette-classic"
5382           },
5383           "custom": {
5384             "axisCenteredZero": false,
5385             "axisColorMode": "text",
5386             "axisLabel": "",
5387             "axisPlacement": "auto",
5388             "barAlignment": 0,
5389             "drawStyle": "line",
5390             "fillOpacity": 10,
5391             "gradientMode": "none",
5392             "hideFrom": {
5393               "legend": false,
5394               "tooltip": false,
5395               "viz": false
5396             },
5397             "lineInterpolation": "linear",
5398             "lineWidth": 1,
5399             "pointSize": 5,
5400             "scaleDistribution": {
5401               "type": "linear"
5402             },
5403             "showPoints": "never",
5404             "spanNulls": true,
5405             "stacking": {
5406               "group": "A",
5407               "mode": "none"
5408             },
5409             "thresholdsStyle": {
5410               "mode": "off"
5411             }
5412           },
5413           "links": [],
5414           "mappings": [],
5415           "thresholds": {
5416             "mode": "absolute",
5417             "steps": [
5418               {
5419                 "color": "green",
5420                 "value": null
5421               },
5422               {
5423                 "color": "red",
5424                 "value": 80
5425               }
5426             ]
5427           },
5428           "unit": "short"
5429         },
5430         "overrides": []
5431       },
5432       "gridPos": {
5433         "h": 8,
5434         "w": 12,
5435         "x": 12,
5436         "y": 134
5437       },
5438       "id": 110,
5439       "options": {
5440         "legend": {
5441           "calcs": [
5442             "mean",
5443             "lastNotNull",
5444             "max"
5445           ],
5446           "displayMode": "table",
5447           "placement": "bottom",
5448           "showLegend": true
5449         },
5450         "tooltip": {
5451           "mode": "single",
5452           "sort": "none"
5453         }
5454       },
5455       "pluginVersion": "8.1.3",
5456       "targets": [
5457         {
5458           "datasource": {
5459             "type": "prometheus",
5460             "uid": "PBFA97CFB590B2093"
5461           },
5462           "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5463           "interval": "",
5464           "legendFormat": "{{instance}}",
5465           "refId": "A"
5466         }
5467       ],
5468       "title": "Connections close rate per instance",
5469       "type": "timeseries"
5470     },
5471     {
5472       "datasource": {
5473         "type": "prometheus",
5474         "uid": "PBFA97CFB590B2093"
5475       },
5476       "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5477       "fieldConfig": {
5478         "defaults": {
5479           "color": {
5480             "mode": "palette-classic"
5481           },
5482           "custom": {
5483             "axisCenteredZero": false,
5484             "axisColorMode": "text",
5485             "axisLabel": "",
5486             "axisPlacement": "auto",
5487             "barAlignment": 0,
5488             "drawStyle": "line",
5489             "fillOpacity": 10,
5490             "gradientMode": "none",
5491             "hideFrom": {
5492               "legend": false,
5493               "tooltip": false,
5494               "viz": false
5495             },
5496             "lineInterpolation": "linear",
5497             "lineWidth": 1,
5498             "pointSize": 5,
5499             "scaleDistribution": {
5500               "type": "linear"
5501             },
5502             "showPoints": "never",
5503             "spanNulls": true,
5504             "stacking": {
5505               "group": "A",
5506               "mode": "none"
5507             },
5508             "thresholdsStyle": {
5509               "mode": "off"
5510             }
5511           },
5512           "links": [],
5513           "mappings": [],
5514           "min": 0,
5515           "thresholds": {
5516             "mode": "absolute",
5517             "steps": [
5518               {
5519                 "color": "green",
5520                 "value": null
5521               },
5522               {
5523                 "color": "red",
5524                 "value": 80
5525               }
5526             ]
5527           },
5528           "unit": "percent"
5529         },
5530         "overrides": []
5531       },
5532       "gridPos": {
5533         "h": 8,
5534         "w": 12,
5535         "x": 0,
5536         "y": 142
5537       },
5538       "id": 112,
5539       "options": {
5540         "legend": {
5541           "calcs": [
5542             "mean",
5543             "lastNotNull",
5544             "max"
5545           ],
5546           "displayMode": "table",
5547           "placement": "bottom",
5548           "showLegend": true
5549         },
5550         "tooltip": {
5551           "mode": "single",
5552           "sort": "none"
5553         }
5554       },
5555       "pluginVersion": "8.1.3",
5556       "targets": [
5557         {
5558           "datasource": {
5559             "type": "prometheus",
5560             "uid": "PBFA97CFB590B2093"
5561           },
5562           "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5563           "interval": "",
5564           "legendFormat": "{{instance}} - {{listener}}",
5565           "refId": "A"
5566         }
5567       ],
5568       "title": "Acceptor Blocked Percentage",
5569       "type": "timeseries"
5570     },
5571     {
5572       "datasource": {
5573         "type": "prometheus",
5574         "uid": "PBFA97CFB590B2093"
5575       },
5576       "fieldConfig": {
5577         "defaults": {
5578           "color": {
5579             "mode": "palette-classic"
5580           },
5581           "custom": {
5582             "axisCenteredZero": false,
5583             "axisColorMode": "text",
5584             "axisLabel": "",
5585             "axisPlacement": "auto",
5586             "barAlignment": 0,
5587             "drawStyle": "line",
5588             "fillOpacity": 10,
5589             "gradientMode": "none",
5590             "hideFrom": {
5591               "legend": false,
5592               "tooltip": false,
5593               "viz": false
5594             },
5595             "lineInterpolation": "linear",
5596             "lineWidth": 1,
5597             "pointSize": 5,
5598             "scaleDistribution": {
5599               "type": "linear"
5600             },
5601             "showPoints": "never",
5602             "spanNulls": true,
5603             "stacking": {
5604               "group": "A",
5605               "mode": "none"
5606             },
5607             "thresholdsStyle": {
5608               "mode": "off"
5609             }
5610           },
5611           "links": [],
5612           "mappings": [],
5613           "thresholds": {
5614             "mode": "absolute",
5615             "steps": [
5616               {
5617                 "color": "green",
5618                 "value": null
5619               },
5620               {
5621                 "color": "red",
5622                 "value": 80
5623               }
5624             ]
5625           },
5626           "unit": "short"
5627         },
5628         "overrides": []
5629       },
5630       "gridPos": {
5631         "h": 8,
5632         "w": 12,
5633         "x": 12,
5634         "y": 142
5635       },
5636       "id": 114,
5637       "options": {
5638         "legend": {
5639           "calcs": [
5640             "mean",
5641             "lastNotNull",
5642             "max"
5643           ],
5644           "displayMode": "table",
5645           "placement": "bottom",
5646           "showLegend": true
5647         },
5648         "tooltip": {
5649           "mode": "single",
5650           "sort": "none"
5651         }
5652       },
5653       "pluginVersion": "8.1.3",
5654       "targets": [
5655         {
5656           "datasource": {
5657             "type": "prometheus",
5658             "uid": "PBFA97CFB590B2093"
5659           },
5660           "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5661           "interval": "",
5662           "legendFormat": "{{client_software_name}} {{client_software_version}}",
5663           "refId": "A"
5664         }
5665       ],
5666       "title": "Connections per client version",
5667       "type": "timeseries"
5668     },
5669     {
5670       "collapsed": false,
5671       "datasource": {
5672         "type": "datasource",
5673         "uid": "grafana"
5674       },
5675       "gridPos": {
5676         "h": 1,
5677         "w": 24,
5678         "x": 0,
5679         "y": 150
5680       },
5681       "id": 120,
5682       "panels": [],
5683       "targets": [
5684         {
5685           "datasource": {
5686             "type": "datasource",
5687             "uid": "grafana"
5688           },
5689           "refId": "A"
5690         }
5691       ],
5692       "title": "Group Coordinator",
5693       "type": "row"
5694     },
5695     {
5696       "datasource": {
5697         "type": "prometheus",
5698         "uid": "PBFA97CFB590B2093"
5699       },
5700       "description": "Number of consumer groups per group coordinator",
5701       "fieldConfig": {
5702         "defaults": {
5703           "color": {
5704             "mode": "palette-classic"
5705           },
5706           "custom": {
5707             "axisCenteredZero": false,
5708             "axisColorMode": "text",
5709             "axisLabel": "",
5710             "axisPlacement": "auto",
5711             "barAlignment": 0,
5712             "drawStyle": "line",
5713             "fillOpacity": 10,
5714             "gradientMode": "none",
5715             "hideFrom": {
5716               "legend": false,
5717               "tooltip": false,
5718               "viz": false
5719             },
5720             "lineInterpolation": "linear",
5721             "lineWidth": 1,
5722             "pointSize": 5,
5723             "scaleDistribution": {
5724               "type": "linear"
5725             },
5726             "showPoints": "never",
5727             "spanNulls": true,
5728             "stacking": {
5729               "group": "A",
5730               "mode": "none"
5731             },
5732             "thresholdsStyle": {
5733               "mode": "off"
5734             }
5735           },
5736           "links": [],
5737           "mappings": [],
5738           "thresholds": {
5739             "mode": "absolute",
5740             "steps": [
5741               {
5742                 "color": "green",
5743                 "value": null
5744               },
5745               {
5746                 "color": "red",
5747                 "value": 80
5748               }
5749             ]
5750           },
5751           "unit": "short"
5752         },
5753         "overrides": []
5754       },
5755       "gridPos": {
5756         "h": 8,
5757         "w": 12,
5758         "x": 0,
5759         "y": 151
5760       },
5761       "id": 116,
5762       "options": {
5763         "legend": {
5764           "calcs": [
5765             "mean",
5766             "lastNotNull",
5767             "max"
5768           ],
5769           "displayMode": "table",
5770           "placement": "bottom",
5771           "showLegend": true
5772         },
5773         "tooltip": {
5774           "mode": "single",
5775           "sort": "none"
5776         }
5777       },
5778       "pluginVersion": "8.1.3",
5779       "targets": [
5780         {
5781           "datasource": {
5782             "type": "prometheus",
5783             "uid": "PBFA97CFB590B2093"
5784           },
5785           "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5786           "instant": false,
5787           "interval": "",
5788           "legendFormat": "{{instance}}",
5789           "refId": "A"
5790         }
5791       ],
5792       "title": "Consumer groups number per coordinator",
5793       "type": "timeseries"
5794     },
5795     {
5796       "datasource": {
5797         "type": "prometheus",
5798         "uid": "PBFA97CFB590B2093"
5799       },
5800       "description": "Number of consumer group per state",
5801       "fieldConfig": {
5802         "defaults": {
5803           "color": {
5804             "mode": "palette-classic"
5805           },
5806           "custom": {
5807             "axisCenteredZero": false,
5808             "axisColorMode": "text",
5809             "axisLabel": "",
5810             "axisPlacement": "auto",
5811             "barAlignment": 0,
5812             "drawStyle": "line",
5813             "fillOpacity": 10,
5814             "gradientMode": "none",
5815             "hideFrom": {
5816               "legend": false,
5817               "tooltip": false,
5818               "viz": false
5819             },
5820             "lineInterpolation": "linear",
5821             "lineWidth": 1,
5822             "pointSize": 5,
5823             "scaleDistribution": {
5824               "type": "linear"
5825             },
5826             "showPoints": "never",
5827             "spanNulls": true,
5828             "stacking": {
5829               "group": "A",
5830               "mode": "none"
5831             },
5832             "thresholdsStyle": {
5833               "mode": "off"
5834             }
5835           },
5836           "links": [],
5837           "mappings": [],
5838           "thresholds": {
5839             "mode": "absolute",
5840             "steps": [
5841               {
5842                 "color": "green",
5843                 "value": null
5844               },
5845               {
5846                 "color": "red",
5847                 "value": 80
5848               }
5849             ]
5850           },
5851           "unit": "short"
5852         },
5853         "overrides": []
5854       },
5855       "gridPos": {
5856         "h": 8,
5857         "w": 12,
5858         "x": 12,
5859         "y": 151
5860       },
5861       "id": 118,
5862       "options": {
5863         "legend": {
5864           "calcs": [
5865             "mean",
5866             "lastNotNull",
5867             "max"
5868           ],
5869           "displayMode": "table",
5870           "placement": "bottom",
5871           "showLegend": true
5872         },
5873         "tooltip": {
5874           "mode": "single",
5875           "sort": "none"
5876         }
5877       },
5878       "pluginVersion": "8.1.3",
5879       "targets": [
5880         {
5881           "datasource": {
5882             "type": "prometheus",
5883             "uid": "PBFA97CFB590B2093"
5884           },
5885           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5886           "instant": false,
5887           "interval": "",
5888           "legendFormat": "stable",
5889           "refId": "A"
5890         },
5891         {
5892           "datasource": {
5893             "type": "prometheus",
5894             "uid": "PBFA97CFB590B2093"
5895           },
5896           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5897           "interval": "",
5898           "legendFormat": "preparing-rebalance",
5899           "refId": "B"
5900         },
5901         {
5902           "datasource": {
5903             "type": "prometheus",
5904             "uid": "PBFA97CFB590B2093"
5905           },
5906           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5907           "interval": "",
5908           "legendFormat": "dead",
5909           "refId": "C"
5910         },
5911         {
5912           "datasource": {
5913             "type": "prometheus",
5914             "uid": "PBFA97CFB590B2093"
5915           },
5916           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5917           "interval": "",
5918           "legendFormat": "completing-rebalance",
5919           "refId": "D"
5920         },
5921         {
5922           "datasource": {
5923             "type": "prometheus",
5924             "uid": "PBFA97CFB590B2093"
5925           },
5926           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5927           "interval": "",
5928           "legendFormat": "empty",
5929           "refId": "E"
5930         }
5931       ],
5932       "title": "Nb consumer groups per state",
5933       "type": "timeseries"
5934     },
5935     {
5936       "collapsed": false,
5937       "datasource": {
5938         "type": "datasource",
5939         "uid": "grafana"
5940       },
5941       "gridPos": {
5942         "h": 1,
5943         "w": 24,
5944         "x": 0,
5945         "y": 159
5946       },
5947       "id": 46,
5948       "panels": [],
5949       "targets": [
5950         {
5951           "datasource": {
5952             "type": "datasource",
5953             "uid": "grafana"
5954           },
5955           "refId": "A"
5956         }
5957       ],
5958       "title": "Message Conversion",
5959       "type": "row"
5960     },
5961     {
5962       "datasource": {
5963         "type": "prometheus",
5964         "uid": "PBFA97CFB590B2093"
5965       },
5966       "description": "The number of messages produced converted to match the log.message.format.version.",
5967       "fieldConfig": {
5968         "defaults": {
5969           "color": {
5970             "mode": "palette-classic"
5971           },
5972           "custom": {
5973             "axisCenteredZero": false,
5974             "axisColorMode": "text",
5975             "axisLabel": "",
5976             "axisPlacement": "auto",
5977             "barAlignment": 0,
5978             "drawStyle": "line",
5979             "fillOpacity": 10,
5980             "gradientMode": "none",
5981             "hideFrom": {
5982               "legend": false,
5983               "tooltip": false,
5984               "viz": false
5985             },
5986             "lineInterpolation": "linear",
5987             "lineWidth": 1,
5988             "pointSize": 5,
5989             "scaleDistribution": {
5990               "type": "linear"
5991             },
5992             "showPoints": "never",
5993             "spanNulls": true,
5994             "stacking": {
5995               "group": "A",
5996               "mode": "none"
5997             },
5998             "thresholdsStyle": {
5999               "mode": "off"
6000             }
6001           },
6002           "links": [],
6003           "mappings": [],
6004           "min": 0,
6005           "thresholds": {
6006             "mode": "absolute",
6007             "steps": [
6008               {
6009                 "color": "green",
6010                 "value": null
6011               },
6012               {
6013                 "color": "red",
6014                 "value": 80
6015               }
6016             ]
6017           },
6018           "unit": "ops"
6019         },
6020         "overrides": []
6021       },
6022       "gridPos": {
6023         "h": 8,
6024         "w": 8,
6025         "x": 0,
6026         "y": 160
6027       },
6028       "id": 48,
6029       "options": {
6030         "legend": {
6031           "calcs": [
6032             "mean",
6033             "lastNotNull",
6034             "max"
6035           ],
6036           "displayMode": "list",
6037           "placement": "bottom",
6038           "showLegend": true
6039         },
6040         "tooltip": {
6041           "mode": "single",
6042           "sort": "none"
6043         }
6044       },
6045       "pluginVersion": "8.1.3",
6046       "targets": [
6047         {
6048           "datasource": {
6049             "type": "prometheus",
6050             "uid": "PBFA97CFB590B2093"
6051           },
6052           "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6053           "hide": false,
6054           "interval": "",
6055           "legendFormat": "{{topic}}",
6056           "refId": "A"
6057         }
6058       ],
6059       "title": "Number of produced message conversion",
6060       "type": "timeseries"
6061     },
6062     {
6063       "datasource": {
6064         "type": "prometheus",
6065         "uid": "PBFA97CFB590B2093"
6066       },
6067       "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
6068       "fieldConfig": {
6069         "defaults": {
6070           "color": {
6071             "mode": "palette-classic"
6072           },
6073           "custom": {
6074             "axisCenteredZero": false,
6075             "axisColorMode": "text",
6076             "axisLabel": "",
6077             "axisPlacement": "auto",
6078             "barAlignment": 0,
6079             "drawStyle": "line",
6080             "fillOpacity": 10,
6081             "gradientMode": "none",
6082             "hideFrom": {
6083               "legend": false,
6084               "tooltip": false,
6085               "viz": false
6086             },
6087             "lineInterpolation": "linear",
6088             "lineWidth": 1,
6089             "pointSize": 5,
6090             "scaleDistribution": {
6091               "type": "linear"
6092             },
6093             "showPoints": "never",
6094             "spanNulls": true,
6095             "stacking": {
6096               "group": "A",
6097               "mode": "none"
6098             },
6099             "thresholdsStyle": {
6100               "mode": "off"
6101             }
6102           },
6103           "links": [],
6104           "mappings": [],
6105           "min": 0,
6106           "thresholds": {
6107             "mode": "absolute",
6108             "steps": [
6109               {
6110                 "color": "green",
6111                 "value": null
6112               },
6113               {
6114                 "color": "red",
6115                 "value": 80
6116               }
6117             ]
6118           },
6119           "unit": "ops"
6120         },
6121         "overrides": []
6122       },
6123       "gridPos": {
6124         "h": 8,
6125         "w": 8,
6126         "x": 8,
6127         "y": 160
6128       },
6129       "id": 51,
6130       "options": {
6131         "legend": {
6132           "calcs": [
6133             "mean",
6134             "lastNotNull",
6135             "max"
6136           ],
6137           "displayMode": "list",
6138           "placement": "bottom",
6139           "showLegend": true
6140         },
6141         "tooltip": {
6142           "mode": "single",
6143           "sort": "none"
6144         }
6145       },
6146       "pluginVersion": "8.1.3",
6147       "targets": [
6148         {
6149           "datasource": {
6150             "type": "prometheus",
6151             "uid": "PBFA97CFB590B2093"
6152           },
6153           "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6154           "hide": false,
6155           "interval": "",
6156           "legendFormat": "{{topic}}",
6157           "refId": "A"
6158         }
6159       ],
6160       "title": "Number of consumed message conversion",
6161       "type": "timeseries"
6162     },
6163     {
6164       "datasource": {
6165         "type": "datasource",
6166         "uid": "grafana"
6167       },
6168       "description": "Number of connection per client version",
6169       "fieldConfig": {
6170         "defaults": {
6171           "color": {
6172             "mode": "palette-classic"
6173           },
6174           "custom": {
6175             "hideFrom": {
6176               "legend": false,
6177               "tooltip": false,
6178               "viz": false
6179             }
6180           },
6181           "decimals": 0,
6182           "mappings": [],
6183           "unit": "short"
6184         },
6185         "overrides": []
6186       },
6187       "gridPos": {
6188         "h": 8,
6189         "w": 8,
6190         "x": 16,
6191         "y": 160
6192       },
6193       "id": 96,
6194       "links": [],
6195       "options": {
6196         "legend": {
6197           "calcs": [],
6198           "displayMode": "table",
6199           "placement": "bottom",
6200           "showLegend": true,
6201           "values": [
6202             "value"
6203           ]
6204         },
6205         "pieType": "pie",
6206         "reduceOptions": {
6207           "calcs": [
6208             "lastNotNull"
6209           ],
6210           "fields": "",
6211           "values": false
6212         },
6213         "tooltip": {
6214           "mode": "single",
6215           "sort": "none"
6216         }
6217       },
6218       "targets": [
6219         {
6220           "datasource": {
6221             "type": "datasource",
6222             "uid": "grafana"
6223           },
6224           "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6225           "interval": "",
6226           "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6227           "refId": "A"
6228         }
6229       ],
6230       "title": "Client version repartition",
6231       "type": "piechart"
6232     }
6233   ],
6234   "refresh": "5s",
6235   "schemaVersion": 38,
6236   "style": "dark",
6237   "tags": [],
6238   "templating": {
6239     "list": [
6240       {
6241         "current": {
6242           "selected": false,
6243           "text": "dev",
6244           "value": "dev"
6245         },
6246         "datasource": {
6247           "type": "prometheus",
6248           "uid": "PBFA97CFB590B2093"
6249         },
6250         "definition": "label_values(env)",
6251         "hide": 0,
6252         "includeAll": false,
6253         "label": "Environment",
6254         "multi": false,
6255         "name": "env",
6256         "options": [],
6257         "query": {
6258           "query": "label_values(env)",
6259           "refId": "Prometheus-env-Variable-Query"
6260         },
6261         "refresh": 1,
6262         "regex": "",
6263         "skipUrlSync": false,
6264         "sort": 0,
6265         "tagValuesQuery": "",
6266         "tagsQuery": "",
6267         "type": "query",
6268         "useTags": false
6269       },
6270       {
6271         "current": {
6272           "selected": true,
6273           "text": [
6274             "All"
6275           ],
6276           "value": [
6277             "$__all"
6278           ]
6279         },
6280         "datasource": {
6281           "type": "prometheus",
6282           "uid": "PBFA97CFB590B2093"
6283         },
6284         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6285         "hide": 0,
6286         "includeAll": true,
6287         "label": "Instance",
6288         "multi": true,
6289         "name": "instance",
6290         "options": [],
6291         "query": {
6292           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6293           "refId": "Prometheus-instance-Variable-Query"
6294         },
6295         "refresh": 1,
6296         "regex": "",
6297         "skipUrlSync": false,
6298         "sort": 0,
6299         "tagValuesQuery": "",
6300         "tagsQuery": "",
6301         "type": "query",
6302         "useTags": false
6303       },
6304       {
6305         "allValue": "",
6306         "current": {
6307           "selected": true,
6308           "text": [
6309             "0.95"
6310           ],
6311           "value": [
6312             "0.95"
6313           ]
6314         },
6315         "datasource": {
6316           "type": "prometheus",
6317           "uid": "PBFA97CFB590B2093"
6318         },
6319         "definition": "label_values(quantile)",
6320         "hide": 0,
6321         "includeAll": true,
6322         "label": "Percentile",
6323         "multi": true,
6324         "name": "percentile",
6325         "options": [],
6326         "query": {
6327           "query": "label_values(quantile)",
6328           "refId": "Prometheus-percentile-Variable-Query"
6329         },
6330         "refresh": 1,
6331         "regex": "",
6332         "skipUrlSync": false,
6333         "sort": 0,
6334         "tagValuesQuery": "",
6335         "tagsQuery": "",
6336         "type": "query",
6337         "useTags": false
6338       }
6339     ]
6340   },
6341   "time": {
6342     "from": "now-5m",
6343     "to": "now"
6344   },
6345   "timepicker": {
6346     "refresh_intervals": [
6347       "5s",
6348       "10s",
6349       "30s",
6350       "1m",
6351       "5m",
6352       "15m",
6353       "30m",
6354       "1h",
6355       "2h",
6356       "1d"
6357     ],
6358     "time_options": [
6359       "5m",
6360       "15m",
6361       "1h",
6362       "6h",
6363       "12h",
6364       "24h",
6365       "2d",
6366       "7d",
6367       "30d"
6368     ]
6369   },
6370   "timezone": "browser",
6371   "title": "Kafka cluster",
6372   "uid": "qu-QZdfZz",
6373   "version": 1,
6374   "weekStart": ""
6375 }