juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of Brokers Online",
  64       "fieldConfig": {
  65         "defaults": {
  66           "color": {
  67             "mode": "thresholds"
  68           },
  69           "mappings": [
  70             {
  71               "options": {
  72                 "match": "null",
  73                 "result": {
  74                   "text": "N/A"
  75                 }
  76               },
  77               "type": "special"
  78             }
  79           ],
  80           "thresholds": {
  81             "mode": "absolute",
  82             "steps": [
  83               {
  84                 "color": "#d44a3a",
  85                 "value": null
  86               },
  87               {
  88                 "color": "rgba(237, 129, 40, 0.89)",
  89                 "value": 0
  90               },
  91               {
  92                 "color": "#299c46",
  93                 "value": 2
  94               }
  95             ]
  96           },
  97           "unit": "none"
  98         },
  99         "overrides": []
 100       },
 101       "gridPos": {
 102         "h": 4,
 103         "w": 4,
 104         "x": 0,
 105         "y": 1
 106       },
 107       "id": 14,
 108       "links": [],
 109       "maxDataPoints": 100,
 110       "options": {
 111         "colorMode": "value",
 112         "graphMode": "none",
 113         "justifyMode": "auto",
 114         "orientation": "horizontal",
 115         "reduceOptions": {
 116           "calcs": [
 117             "lastNotNull"
 118           ],
 119           "fields": "",
 120           "values": false
 121         },
 122         "text": {},
 123         "textMode": "auto"
 124       },
 125       "pluginVersion": "10.0.3",
 126       "repeatDirection": "h",
 127       "targets": [
 128         {
 129           "datasource": {
 130             "type": "prometheus",
 131             "uid": "PBFA97CFB590B2093"
 132           },
 133           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 134           "format": "time_series",
 135           "instant": true,
 136           "interval": "",
 137           "intervalFactor": 1,
 138           "legendFormat": "",
 139           "refId": "A"
 140         }
 141       ],
 142       "title": "Brokers Online",
 143       "type": "stat"
 144     },
 145     {
 146       "datasource": {
 147         "type": "prometheus",
 148         "uid": "PBFA97CFB590B2093"
 149       },
 150       "description": "Number of active controllers in the cluster.",
 151       "fieldConfig": {
 152         "defaults": {
 153           "mappings": [
 154             {
 155               "options": {
 156                 "match": "null",
 157                 "result": {
 158                   "text": "N/A"
 159                 }
 160               },
 161               "type": "special"
 162             }
 163           ],
 164           "thresholds": {
 165             "mode": "absolute",
 166             "steps": [
 167               {
 168                 "color": "#299c46",
 169                 "value": null
 170               },
 171               {
 172                 "color": "#e5ac0e",
 173                 "value": 2
 174               },
 175               {
 176                 "color": "#bf1b00"
 177               }
 178             ]
 179           },
 180           "unit": "none"
 181         },
 182         "overrides": []
 183       },
 184       "gridPos": {
 185         "h": 4,
 186         "w": 5,
 187         "x": 4,
 188         "y": 1
 189       },
 190       "id": 12,
 191       "links": [],
 192       "maxDataPoints": 100,
 193       "options": {
 194         "colorMode": "value",
 195         "graphMode": "none",
 196         "justifyMode": "auto",
 197         "orientation": "vertical",
 198         "reduceOptions": {
 199           "calcs": [
 200             "lastNotNull"
 201           ],
 202           "fields": "",
 203           "values": false
 204         },
 205         "text": {},
 206         "textMode": "value_and_name"
 207       },
 208       "pluginVersion": "10.0.3",
 209       "targets": [
 210         {
 211           "datasource": {
 212             "type": "prometheus",
 213             "uid": "PBFA97CFB590B2093"
 214           },
 215           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 216           "format": "time_series",
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 1,
 220           "legendFormat": "{{instance}}",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Active Controllers",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Partitions that are online",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#d44a3a",
 254                 "value": null
 255               },
 256               {
 257                 "color": "rgba(237, 129, 40, 0.89)",
 258                 "value": 0
 259               },
 260               {
 261                 "color": "#299c46",
 262                 "value": 0
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 6,
 273         "x": 9,
 274         "y": 1
 275       },
 276       "id": 18,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Online Partitions",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 15,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Unclean leader election rate",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#299c46",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 2
 430               },
 431               {
 432                 "color": "#d44a3a"
 433               }
 434             ]
 435           },
 436           "unit": "none"
 437         },
 438         "overrides": []
 439       },
 440       "gridPos": {
 441         "h": 4,
 442         "w": 4,
 443         "x": 19,
 444         "y": 1
 445       },
 446       "id": 16,
 447       "links": [],
 448       "maxDataPoints": 100,
 449       "options": {
 450         "colorMode": "value",
 451         "graphMode": "none",
 452         "justifyMode": "auto",
 453         "orientation": "horizontal",
 454         "reduceOptions": {
 455           "calcs": [
 456             "lastNotNull"
 457           ],
 458           "fields": "",
 459           "values": false
 460         },
 461         "text": {},
 462         "textMode": "auto"
 463       },
 464       "pluginVersion": "10.0.3",
 465       "targets": [
 466         {
 467           "datasource": {
 468             "type": "prometheus",
 469             "uid": "PBFA97CFB590B2093"
 470           },
 471           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 472           "format": "time_series",
 473           "instant": true,
 474           "interval": "",
 475           "intervalFactor": 1,
 476           "legendFormat": "",
 477           "refId": "A"
 478         }
 479       ],
 480       "title": "Unclean Leader Election Rate",
 481       "type": "stat"
 482     },
 483     {
 484       "datasource": {
 485         "type": "prometheus",
 486         "uid": "PBFA97CFB590B2093"
 487       },
 488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 489       "fieldConfig": {
 490         "defaults": {
 491           "color": {
 492             "mode": "thresholds"
 493           },
 494           "mappings": [
 495             {
 496               "options": {
 497                 "match": "null",
 498                 "result": {
 499                   "text": "N/A"
 500                 }
 501               },
 502               "type": "special"
 503             }
 504           ],
 505           "thresholds": {
 506             "mode": "absolute",
 507             "steps": [
 508               {
 509                 "color": "#508642",
 510                 "value": null
 511               },
 512               {
 513                 "color": "rgba(237, 129, 40, 0.89)",
 514                 "value": 1
 515               },
 516               {
 517                 "color": "#bf1b00",
 518                 "value": 5
 519               }
 520             ]
 521           },
 522           "unit": "none"
 523         },
 524         "overrides": []
 525       },
 526       "gridPos": {
 527         "h": 4,
 528         "w": 4,
 529         "x": 0,
 530         "y": 5
 531       },
 532       "id": 20,
 533       "links": [],
 534       "maxDataPoints": 100,
 535       "options": {
 536         "colorMode": "value",
 537         "graphMode": "none",
 538         "justifyMode": "auto",
 539         "orientation": "horizontal",
 540         "reduceOptions": {
 541           "calcs": [
 542             "lastNotNull"
 543           ],
 544           "fields": "",
 545           "values": false
 546         },
 547         "text": {},
 548         "textMode": "auto"
 549       },
 550       "pluginVersion": "10.0.3",
 551       "targets": [
 552         {
 553           "datasource": {
 554             "type": "prometheus",
 555             "uid": "PBFA97CFB590B2093"
 556           },
 557           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 558           "format": "time_series",
 559           "hide": false,
 560           "instant": true,
 561           "interval": "",
 562           "intervalFactor": 2,
 563           "legendFormat": "",
 564           "refId": "A"
 565         }
 566       ],
 567       "title": "Under Replicated Partitions",
 568       "type": "stat"
 569     },
 570     {
 571       "datasource": {
 572         "type": "prometheus",
 573         "uid": "PBFA97CFB590B2093"
 574       },
 575       "description": "",
 576       "fieldConfig": {
 577         "defaults": {
 578           "color": {
 579             "mode": "palette-classic"
 580           },
 581           "custom": {
 582             "axisCenteredZero": false,
 583             "axisColorMode": "text",
 584             "axisLabel": "",
 585             "axisPlacement": "auto",
 586             "barAlignment": 0,
 587             "drawStyle": "line",
 588             "fillOpacity": 10,
 589             "gradientMode": "none",
 590             "hideFrom": {
 591               "legend": false,
 592               "tooltip": false,
 593               "viz": false
 594             },
 595             "lineInterpolation": "linear",
 596             "lineWidth": 1,
 597             "pointSize": 5,
 598             "scaleDistribution": {
 599               "type": "linear"
 600             },
 601             "showPoints": "never",
 602             "spanNulls": true,
 603             "stacking": {
 604               "group": "A",
 605               "mode": "none"
 606             },
 607             "thresholdsStyle": {
 608               "mode": "off"
 609             }
 610           },
 611           "decimals": 0,
 612           "links": [],
 613           "mappings": [],
 614           "min": 0,
 615           "thresholds": {
 616             "mode": "absolute",
 617             "steps": [
 618               {
 619                 "color": "green",
 620                 "value": null
 621               },
 622               {
 623                 "color": "red",
 624                 "value": 80
 625               }
 626             ]
 627           },
 628           "unit": "partitions"
 629         },
 630         "overrides": []
 631       },
 632       "gridPos": {
 633         "h": 12,
 634         "w": 11,
 635         "x": 4,
 636         "y": 5
 637       },
 638       "id": 123,
 639       "options": {
 640         "legend": {
 641           "calcs": [
 642             "lastNotNull",
 643             "max"
 644           ],
 645           "displayMode": "table",
 646           "placement": "bottom",
 647           "showLegend": true
 648         },
 649         "tooltip": {
 650           "mode": "single",
 651           "sort": "none"
 652         }
 653       },
 654       "pluginVersion": "8.1.3",
 655       "targets": [
 656         {
 657           "datasource": {
 658             "type": "prometheus",
 659             "uid": "PBFA97CFB590B2093"
 660           },
 661           "editorMode": "code",
 662           "exemplar": false,
 663           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 664           "hide": false,
 665           "interval": "",
 666           "legendFormat": "{{instance}}",
 667           "range": true,
 668           "refId": "A"
 669         }
 670       ],
 671       "title": "Underreplicated Partitions",
 672       "type": "timeseries"
 673     },
 674     {
 675       "datasource": {
 676         "type": "prometheus",
 677         "uid": "PBFA97CFB590B2093"
 678       },
 679       "description": "",
 680       "fieldConfig": {
 681         "defaults": {
 682           "color": {
 683             "mode": "palette-classic"
 684           },
 685           "custom": {
 686             "hideFrom": {
 687               "legend": false,
 688               "tooltip": false,
 689               "viz": false
 690             }
 691           },
 692           "decimals": 0,
 693           "links": [],
 694           "mappings": [
 695             {
 696               "options": {
 697                 "null": {
 698                   "index": 0,
 699                   "text": "0"
 700                 }
 701               },
 702               "type": "value"
 703             }
 704           ],
 705           "min": 0,
 706           "unit": "partitions"
 707         },
 708         "overrides": []
 709       },
 710       "gridPos": {
 711         "h": 12,
 712         "w": 8,
 713         "x": 15,
 714         "y": 5
 715       },
 716       "id": 124,
 717       "options": {
 718         "legend": {
 719           "displayMode": "list",
 720           "placement": "right",
 721           "showLegend": true
 722         },
 723         "pieType": "pie",
 724         "reduceOptions": {
 725           "calcs": [
 726             "lastNotNull"
 727           ],
 728           "fields": "",
 729           "values": false
 730         },
 731         "tooltip": {
 732           "mode": "single",
 733           "sort": "none"
 734         }
 735       },
 736       "pluginVersion": "8.1.3",
 737       "targets": [
 738         {
 739           "datasource": {
 740             "type": "prometheus",
 741             "uid": "PBFA97CFB590B2093"
 742           },
 743           "editorMode": "code",
 744           "exemplar": false,
 745           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 746           "hide": false,
 747           "interval": "",
 748           "legendFormat": "{{instance}}",
 749           "range": true,
 750           "refId": "A"
 751         }
 752       ],
 753       "title": "Leader Count",
 754       "type": "piechart"
 755     },
 756     {
 757       "datasource": {
 758         "type": "prometheus",
 759         "uid": "PBFA97CFB590B2093"
 760       },
 761       "description": "Number of partitions under min insync replicas.",
 762       "fieldConfig": {
 763         "defaults": {
 764           "color": {
 765             "mode": "thresholds"
 766           },
 767           "mappings": [
 768             {
 769               "options": {
 770                 "match": "null",
 771                 "result": {
 772                   "text": "N/A"
 773                 }
 774               },
 775               "type": "special"
 776             }
 777           ],
 778           "thresholds": {
 779             "mode": "absolute",
 780             "steps": [
 781               {
 782                 "color": "#508642",
 783                 "value": null
 784               },
 785               {
 786                 "color": "rgba(237, 129, 40, 0.89)",
 787                 "value": 1
 788               },
 789               {
 790                 "color": "#bf1b00",
 791                 "value": 5
 792               }
 793             ]
 794           },
 795           "unit": "none"
 796         },
 797         "overrides": []
 798       },
 799       "gridPos": {
 800         "h": 4,
 801         "w": 4,
 802         "x": 0,
 803         "y": 9
 804       },
 805       "id": 32,
 806       "links": [],
 807       "maxDataPoints": 100,
 808       "options": {
 809         "colorMode": "value",
 810         "graphMode": "none",
 811         "justifyMode": "auto",
 812         "orientation": "horizontal",
 813         "reduceOptions": {
 814           "calcs": [
 815             "lastNotNull"
 816           ],
 817           "fields": "",
 818           "values": false
 819         },
 820         "text": {},
 821         "textMode": "auto"
 822       },
 823       "pluginVersion": "10.0.3",
 824       "targets": [
 825         {
 826           "datasource": {
 827             "type": "prometheus",
 828             "uid": "PBFA97CFB590B2093"
 829           },
 830           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 831           "format": "time_series",
 832           "hide": false,
 833           "instant": true,
 834           "interval": "",
 835           "intervalFactor": 2,
 836           "legendFormat": "",
 837           "refId": "A"
 838         }
 839       ],
 840       "title": "Under Min ISR Partitions",
 841       "type": "stat"
 842     },
 843     {
 844       "datasource": {
 845         "type": "prometheus",
 846         "uid": "PBFA97CFB590B2093"
 847       },
 848       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 849       "fieldConfig": {
 850         "defaults": {
 851           "color": {
 852             "mode": "thresholds"
 853           },
 854           "mappings": [
 855             {
 856               "options": {
 857                 "match": "null",
 858                 "result": {
 859                   "text": "N/A"
 860                 }
 861               },
 862               "type": "special"
 863             }
 864           ],
 865           "thresholds": {
 866             "mode": "absolute",
 867             "steps": [
 868               {
 869                 "color": "#508642",
 870                 "value": null
 871               },
 872               {
 873                 "color": "#ef843c",
 874                 "value": 1
 875               },
 876               {
 877                 "color": "#bf1b00",
 878                 "value": 1
 879               }
 880             ]
 881           },
 882           "unit": "none"
 883         },
 884         "overrides": []
 885       },
 886       "gridPos": {
 887         "h": 4,
 888         "w": 4,
 889         "x": 0,
 890         "y": 13
 891       },
 892       "id": 22,
 893       "links": [],
 894       "maxDataPoints": 100,
 895       "options": {
 896         "colorMode": "value",
 897         "graphMode": "none",
 898         "justifyMode": "auto",
 899         "orientation": "horizontal",
 900         "reduceOptions": {
 901           "calcs": [
 902             "lastNotNull"
 903           ],
 904           "fields": "",
 905           "values": false
 906         },
 907         "text": {},
 908         "textMode": "auto"
 909       },
 910       "pluginVersion": "10.0.3",
 911       "targets": [
 912         {
 913           "datasource": {
 914             "type": "prometheus",
 915             "uid": "PBFA97CFB590B2093"
 916           },
 917           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 918           "format": "time_series",
 919           "instant": true,
 920           "interval": "",
 921           "intervalFactor": 1,
 922           "legendFormat": "",
 923           "refId": "A"
 924         }
 925       ],
 926       "title": "Offline Partitions Count",
 927       "type": "stat"
 928     },
 929     {
 930       "collapsed": true,
 931       "datasource": {
 932         "type": "datasource",
 933         "uid": "grafana"
 934       },
 935       "gridPos": {
 936         "h": 1,
 937         "w": 24,
 938         "x": 0,
 939         "y": 17
 940       },
 941       "id": 40,
 942       "panels": [
 943         {
 944           "datasource": {
 945             "type": "prometheus",
 946             "uid": "PBFA97CFB590B2093"
 947           },
 948           "fieldConfig": {
 949             "defaults": {
 950               "color": {
 951                 "mode": "palette-classic"
 952               },
 953               "custom": {
 954                 "axisCenteredZero": false,
 955                 "axisColorMode": "text",
 956                 "axisLabel": "Cores",
 957                 "axisPlacement": "auto",
 958                 "barAlignment": 0,
 959                 "drawStyle": "line",
 960                 "fillOpacity": 10,
 961                 "gradientMode": "none",
 962                 "hideFrom": {
 963                   "legend": false,
 964                   "tooltip": false,
 965                   "viz": false
 966                 },
 967                 "lineInterpolation": "linear",
 968                 "lineWidth": 2,
 969                 "pointSize": 5,
 970                 "scaleDistribution": {
 971                   "type": "linear"
 972                 },
 973                 "showPoints": "never",
 974                 "spanNulls": false,
 975                 "stacking": {
 976                   "group": "A",
 977                   "mode": "none"
 978                 },
 979                 "thresholdsStyle": {
 980                   "mode": "off"
 981                 }
 982               },
 983               "links": [],
 984               "mappings": [],
 985               "thresholds": {
 986                 "mode": "absolute",
 987                 "steps": [
 988                   {
 989                     "color": "green"
 990                   },
 991                   {
 992                     "color": "red",
 993                     "value": 80
 994                   }
 995                 ]
 996               },
 997               "unit": "percent"
 998             },
 999             "overrides": [
1000               {
1001                 "matcher": {
1002                   "id": "byName",
1003                   "options": "localhost:7071"
1004                 },
1005                 "properties": [
1006                   {
1007                     "id": "color",
1008                     "value": {
1009                       "fixedColor": "#629E51",
1010                       "mode": "fixed"
1011                     }
1012                   }
1013                 ]
1014               }
1015             ]
1016           },
1017           "gridPos": {
1018             "h": 12,
1019             "w": 8,
1020             "x": 0,
1021             "y": 19
1022           },
1023           "id": 27,
1024           "links": [],
1025           "options": {
1026             "legend": {
1027               "calcs": [
1028                 "mean",
1029                 "lastNotNull",
1030                 "max"
1031               ],
1032               "displayMode": "table",
1033               "placement": "bottom",
1034               "showLegend": true
1035             },
1036             "tooltip": {
1037               "mode": "single",
1038               "sort": "none"
1039             }
1040           },
1041           "pluginVersion": "8.1.3",
1042           "targets": [
1043             {
1044               "datasource": {
1045                 "type": "prometheus",
1046                 "uid": "PBFA97CFB590B2093"
1047               },
1048               "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1049               "format": "time_series",
1050               "interval": "",
1051               "intervalFactor": 2,
1052               "legendFormat": "{{instance}}",
1053               "metric": "process_cpu_secondspersec",
1054               "refId": "A",
1055               "step": 4
1056             }
1057           ],
1058           "title": "CPU Usage",
1059           "type": "timeseries"
1060         },
1061         {
1062           "datasource": {
1063             "type": "prometheus",
1064             "uid": "PBFA97CFB590B2093"
1065           },
1066           "fieldConfig": {
1067             "defaults": {
1068               "color": {
1069                 "mode": "palette-classic"
1070               },
1071               "custom": {
1072                 "axisCenteredZero": false,
1073                 "axisColorMode": "text",
1074                 "axisLabel": "Memory",
1075                 "axisPlacement": "auto",
1076                 "barAlignment": 0,
1077                 "drawStyle": "line",
1078                 "fillOpacity": 10,
1079                 "gradientMode": "none",
1080                 "hideFrom": {
1081                   "legend": false,
1082                   "tooltip": false,
1083                   "viz": false
1084                 },
1085                 "lineInterpolation": "linear",
1086                 "lineWidth": 2,
1087                 "pointSize": 5,
1088                 "scaleDistribution": {
1089                   "type": "linear"
1090                 },
1091                 "showPoints": "never",
1092                 "spanNulls": false,
1093                 "stacking": {
1094                   "group": "A",
1095                   "mode": "none"
1096                 },
1097                 "thresholdsStyle": {
1098                   "mode": "off"
1099                 }
1100               },
1101               "links": [],
1102               "mappings": [],
1103               "thresholds": {
1104                 "mode": "absolute",
1105                 "steps": [
1106                   {
1107                     "color": "green"
1108                   },
1109                   {
1110                     "color": "red",
1111                     "value": 80
1112                   }
1113                 ]
1114               },
1115               "unit": "bytes"
1116             },
1117             "overrides": [
1118               {
1119                 "matcher": {
1120                   "id": "byName",
1121                   "options": "localhost:7071"
1122                 },
1123                 "properties": [
1124                   {
1125                     "id": "color",
1126                     "value": {
1127                       "fixedColor": "#BA43A9",
1128                       "mode": "fixed"
1129                     }
1130                   }
1131                 ]
1132               },
1133               {
1134                 "__systemRef": "hideSeriesFrom",
1135                 "matcher": {
1136                   "id": "byNames",
1137                   "options": {
1138                     "mode": "exclude",
1139                     "names": [
1140                       "kafka-2:80"
1141                     ],
1142                     "prefix": "All except:",
1143                     "readOnly": true
1144                   }
1145                 },
1146                 "properties": [
1147                   {
1148                     "id": "custom.hideFrom",
1149                     "value": {
1150                       "legend": false,
1151                       "tooltip": false,
1152                       "viz": true
1153                     }
1154                   }
1155                 ]
1156               }
1157             ]
1158           },
1159           "gridPos": {
1160             "h": 12,
1161             "w": 8,
1162             "x": 8,
1163             "y": 19
1164           },
1165           "id": 2,
1166           "links": [],
1167           "options": {
1168             "legend": {
1169               "calcs": [
1170                 "mean",
1171                 "lastNotNull",
1172                 "max"
1173               ],
1174               "displayMode": "table",
1175               "placement": "bottom",
1176               "showLegend": true
1177             },
1178             "tooltip": {
1179               "mode": "single",
1180               "sort": "none"
1181             }
1182           },
1183           "pluginVersion": "8.1.3",
1184           "targets": [
1185             {
1186               "datasource": {
1187                 "type": "prometheus",
1188                 "uid": "PBFA97CFB590B2093"
1189               },
1190               "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1191               "interval": "",
1192               "intervalFactor": 2,
1193               "legendFormat": "{{instance}}",
1194               "metric": "jvm_memory_bytes_used",
1195               "refId": "A",
1196               "step": 4
1197             },
1198             {
1199               "datasource": {
1200                 "type": "prometheus",
1201                 "uid": "PBFA97CFB590B2093"
1202               },
1203               "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1204               "interval": "",
1205               "legendFormat": "{{instance}}",
1206               "refId": "B"
1207             }
1208           ],
1209           "title": "JVM Memory Used",
1210           "type": "timeseries"
1211         },
1212         {
1213           "datasource": {
1214             "type": "prometheus",
1215             "uid": "PBFA97CFB590B2093"
1216           },
1217           "fieldConfig": {
1218             "defaults": {
1219               "color": {
1220                 "mode": "palette-classic"
1221               },
1222               "custom": {
1223                 "axisCenteredZero": false,
1224                 "axisColorMode": "text",
1225                 "axisLabel": "% time in GC",
1226                 "axisPlacement": "auto",
1227                 "barAlignment": 0,
1228                 "drawStyle": "line",
1229                 "fillOpacity": 10,
1230                 "gradientMode": "none",
1231                 "hideFrom": {
1232                   "legend": false,
1233                   "tooltip": false,
1234                   "viz": false
1235                 },
1236                 "lineInterpolation": "linear",
1237                 "lineWidth": 2,
1238                 "pointSize": 5,
1239                 "scaleDistribution": {
1240                   "type": "linear"
1241                 },
1242                 "showPoints": "never",
1243                 "spanNulls": false,
1244                 "stacking": {
1245                   "group": "A",
1246                   "mode": "none"
1247                 },
1248                 "thresholdsStyle": {
1249                   "mode": "off"
1250                 }
1251               },
1252               "links": [],
1253               "mappings": [],
1254               "thresholds": {
1255                 "mode": "absolute",
1256                 "steps": [
1257                   {
1258                     "color": "green"
1259                   },
1260                   {
1261                     "color": "red",
1262                     "value": 80
1263                   }
1264                 ]
1265               },
1266               "unit": "percentunit"
1267             },
1268             "overrides": [
1269               {
1270                 "matcher": {
1271                   "id": "byName",
1272                   "options": "localhost:7071"
1273                 },
1274                 "properties": [
1275                   {
1276                     "id": "color",
1277                     "value": {
1278                       "fixedColor": "#890F02",
1279                       "mode": "fixed"
1280                     }
1281                   }
1282                 ]
1283               }
1284             ]
1285           },
1286           "gridPos": {
1287             "h": 12,
1288             "w": 8,
1289             "x": 16,
1290             "y": 19
1291           },
1292           "id": 3,
1293           "links": [],
1294           "options": {
1295             "legend": {
1296               "calcs": [
1297                 "mean",
1298                 "lastNotNull",
1299                 "max"
1300               ],
1301               "displayMode": "table",
1302               "placement": "bottom",
1303               "showLegend": true
1304             },
1305             "tooltip": {
1306               "mode": "single",
1307               "sort": "none"
1308             }
1309           },
1310           "pluginVersion": "8.1.3",
1311           "targets": [
1312             {
1313               "datasource": {
1314                 "type": "prometheus",
1315                 "uid": "PBFA97CFB590B2093"
1316               },
1317               "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1318               "interval": "",
1319               "intervalFactor": 2,
1320               "legendFormat": "{{instance}}",
1321               "metric": "jvm_gc_collection_seconds_sum",
1322               "refId": "A",
1323               "step": 4
1324             }
1325           ],
1326           "title": "Time spent in GC",
1327           "type": "timeseries"
1328         }
1329       ],
1330       "targets": [
1331         {
1332           "datasource": {
1333             "type": "datasource",
1334             "uid": "grafana"
1335           },
1336           "refId": "A"
1337         }
1338       ],
1339       "title": "System",
1340       "type": "row"
1341     },
1342     {
1343       "collapsed": false,
1344       "datasource": {
1345         "type": "datasource",
1346         "uid": "grafana"
1347       },
1348       "gridPos": {
1349         "h": 1,
1350         "w": 24,
1351         "x": 0,
1352         "y": 18
1353       },
1354       "id": 31,
1355       "panels": [],
1356       "targets": [
1357         {
1358           "datasource": {
1359             "type": "datasource",
1360             "uid": "grafana"
1361           },
1362           "refId": "A"
1363         }
1364       ],
1365       "title": "Throughput & Request rate",
1366       "type": "row"
1367     },
1368     {
1369       "datasource": {
1370         "type": "prometheus",
1371         "uid": "PBFA97CFB590B2093"
1372       },
1373       "fieldConfig": {
1374         "defaults": {
1375           "color": {
1376             "mode": "palette-classic"
1377           },
1378           "custom": {
1379             "axisCenteredZero": false,
1380             "axisColorMode": "text",
1381             "axisLabel": "Bytes/s",
1382             "axisPlacement": "auto",
1383             "barAlignment": 0,
1384             "drawStyle": "line",
1385             "fillOpacity": 10,
1386             "gradientMode": "none",
1387             "hideFrom": {
1388               "legend": false,
1389               "tooltip": false,
1390               "viz": false
1391             },
1392             "lineInterpolation": "linear",
1393             "lineWidth": 2,
1394             "pointSize": 5,
1395             "scaleDistribution": {
1396               "type": "linear"
1397             },
1398             "showPoints": "never",
1399             "spanNulls": false,
1400             "stacking": {
1401               "group": "A",
1402               "mode": "none"
1403             },
1404             "thresholdsStyle": {
1405               "mode": "off"
1406             }
1407           },
1408           "links": [],
1409           "mappings": [],
1410           "thresholds": {
1411             "mode": "absolute",
1412             "steps": [
1413               {
1414                 "color": "green",
1415                 "value": null
1416               },
1417               {
1418                 "color": "red",
1419                 "value": 80
1420               }
1421             ]
1422           },
1423           "unit": "Bps"
1424         },
1425         "overrides": []
1426       },
1427       "gridPos": {
1428         "h": 8,
1429         "w": 12,
1430         "x": 0,
1431         "y": 19
1432       },
1433       "id": 84,
1434       "links": [],
1435       "options": {
1436         "legend": {
1437           "calcs": [
1438             "mean",
1439             "lastNotNull",
1440             "max"
1441           ],
1442           "displayMode": "list",
1443           "placement": "bottom",
1444           "showLegend": true
1445         },
1446         "tooltip": {
1447           "mode": "single",
1448           "sort": "none"
1449         }
1450       },
1451       "pluginVersion": "8.1.3",
1452       "targets": [
1453         {
1454           "datasource": {
1455             "type": "prometheus",
1456             "uid": "PBFA97CFB590B2093"
1457           },
1458           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1459           "format": "time_series",
1460           "interval": "",
1461           "intervalFactor": 2,
1462           "legendFormat": "Bytes in",
1463           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1464           "refId": "A",
1465           "step": 4
1466         },
1467         {
1468           "datasource": {
1469             "type": "prometheus",
1470             "uid": "PBFA97CFB590B2093"
1471           },
1472           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1473           "format": "time_series",
1474           "hide": false,
1475           "interval": "",
1476           "intervalFactor": 2,
1477           "legendFormat": "Bytes out",
1478           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1479           "refId": "B",
1480           "step": 4
1481         }
1482       ],
1483       "title": "Broker network throughput",
1484       "type": "timeseries"
1485     },
1486     {
1487       "datasource": {
1488         "type": "prometheus",
1489         "uid": "PBFA97CFB590B2093"
1490       },
1491       "description": "Produce request rate.",
1492       "fieldConfig": {
1493         "defaults": {
1494           "mappings": [
1495             {
1496               "options": {
1497                 "match": "null",
1498                 "result": {
1499                   "text": "N/A"
1500                 }
1501               },
1502               "type": "special"
1503             }
1504           ],
1505           "thresholds": {
1506             "mode": "absolute",
1507             "steps": [
1508               {
1509                 "color": "green",
1510                 "value": null
1511               }
1512             ]
1513           },
1514           "unit": "none"
1515         },
1516         "overrides": []
1517       },
1518       "gridPos": {
1519         "h": 4,
1520         "w": 4,
1521         "x": 12,
1522         "y": 19
1523       },
1524       "id": 93,
1525       "links": [],
1526       "maxDataPoints": 100,
1527       "options": {
1528         "colorMode": "value",
1529         "graphMode": "area",
1530         "justifyMode": "auto",
1531         "orientation": "horizontal",
1532         "reduceOptions": {
1533           "calcs": [
1534             "lastNotNull"
1535           ],
1536           "fields": "",
1537           "values": false
1538         },
1539         "text": {},
1540         "textMode": "auto"
1541       },
1542       "pluginVersion": "10.0.3",
1543       "targets": [
1544         {
1545           "datasource": {
1546             "type": "prometheus",
1547             "uid": "PBFA97CFB590B2093"
1548           },
1549           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1550           "interval": "",
1551           "legendFormat": "",
1552           "refId": "A"
1553         }
1554       ],
1555       "title": "All Request Per Sec",
1556       "type": "stat"
1557     },
1558     {
1559       "datasource": {
1560         "type": "prometheus",
1561         "uid": "PBFA97CFB590B2093"
1562       },
1563       "description": "Fetch request rate.",
1564       "fieldConfig": {
1565         "defaults": {
1566           "mappings": [
1567             {
1568               "options": {
1569                 "match": "null",
1570                 "result": {
1571                   "text": "N/A"
1572                 }
1573               },
1574               "type": "special"
1575             }
1576           ],
1577           "thresholds": {
1578             "mode": "absolute",
1579             "steps": [
1580               {
1581                 "color": "green",
1582                 "value": null
1583               }
1584             ]
1585           },
1586           "unit": "none"
1587         },
1588         "overrides": []
1589       },
1590       "gridPos": {
1591         "h": 4,
1592         "w": 4,
1593         "x": 16,
1594         "y": 19
1595       },
1596       "id": 37,
1597       "links": [],
1598       "maxDataPoints": 100,
1599       "options": {
1600         "colorMode": "value",
1601         "graphMode": "area",
1602         "justifyMode": "auto",
1603         "orientation": "horizontal",
1604         "reduceOptions": {
1605           "calcs": [
1606             "lastNotNull"
1607           ],
1608           "fields": "",
1609           "values": false
1610         },
1611         "text": {},
1612         "textMode": "auto"
1613       },
1614       "pluginVersion": "10.0.3",
1615       "targets": [
1616         {
1617           "datasource": {
1618             "type": "prometheus",
1619             "uid": "PBFA97CFB590B2093"
1620           },
1621           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1622           "interval": "",
1623           "legendFormat": "",
1624           "refId": "A"
1625         }
1626       ],
1627       "title": "Consumer Fetch Request Per Sec",
1628       "type": "stat"
1629     },
1630     {
1631       "datasource": {
1632         "type": "prometheus",
1633         "uid": "PBFA97CFB590B2093"
1634       },
1635       "description": "Produce request rate.",
1636       "fieldConfig": {
1637         "defaults": {
1638           "mappings": [
1639             {
1640               "options": {
1641                 "match": "null",
1642                 "result": {
1643                   "text": "N/A"
1644                 }
1645               },
1646               "type": "special"
1647             }
1648           ],
1649           "thresholds": {
1650             "mode": "absolute",
1651             "steps": [
1652               {
1653                 "color": "green",
1654                 "value": null
1655               }
1656             ]
1657           },
1658           "unit": "none"
1659         },
1660         "overrides": []
1661       },
1662       "gridPos": {
1663         "h": 4,
1664         "w": 4,
1665         "x": 20,
1666         "y": 19
1667       },
1668       "id": 35,
1669       "links": [],
1670       "maxDataPoints": 100,
1671       "options": {
1672         "colorMode": "value",
1673         "graphMode": "area",
1674         "justifyMode": "auto",
1675         "orientation": "horizontal",
1676         "reduceOptions": {
1677           "calcs": [
1678             "lastNotNull"
1679           ],
1680           "fields": "",
1681           "values": false
1682         },
1683         "text": {},
1684         "textMode": "auto"
1685       },
1686       "pluginVersion": "10.0.3",
1687       "targets": [
1688         {
1689           "datasource": {
1690             "type": "prometheus",
1691             "uid": "PBFA97CFB590B2093"
1692           },
1693           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1694           "interval": "",
1695           "legendFormat": "",
1696           "refId": "A"
1697         }
1698       ],
1699       "title": "Produce Request Per Sec",
1700       "type": "stat"
1701     },
1702     {
1703       "datasource": {
1704         "type": "prometheus",
1705         "uid": "PBFA97CFB590B2093"
1706       },
1707       "description": "Metadata request rate.",
1708       "fieldConfig": {
1709         "defaults": {
1710           "mappings": [
1711             {
1712               "options": {
1713                 "match": "null",
1714                 "result": {
1715                   "text": "N/A"
1716                 }
1717               },
1718               "type": "special"
1719             }
1720           ],
1721           "thresholds": {
1722             "mode": "absolute",
1723             "steps": [
1724               {
1725                 "color": "green",
1726                 "value": null
1727               }
1728             ]
1729           },
1730           "unit": "none"
1731         },
1732         "overrides": []
1733       },
1734       "gridPos": {
1735         "h": 4,
1736         "w": 4,
1737         "x": 12,
1738         "y": 23
1739       },
1740       "id": 36,
1741       "links": [],
1742       "maxDataPoints": 100,
1743       "options": {
1744         "colorMode": "value",
1745         "graphMode": "area",
1746         "justifyMode": "auto",
1747         "orientation": "horizontal",
1748         "reduceOptions": {
1749           "calcs": [
1750             "lastNotNull"
1751           ],
1752           "fields": "",
1753           "values": false
1754         },
1755         "text": {},
1756         "textMode": "auto"
1757       },
1758       "pluginVersion": "10.0.3",
1759       "targets": [
1760         {
1761           "datasource": {
1762             "type": "prometheus",
1763             "uid": "PBFA97CFB590B2093"
1764           },
1765           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1766           "interval": "",
1767           "legendFormat": "",
1768           "refId": "A"
1769         }
1770       ],
1771       "title": "Metadata Request Per Sec",
1772       "type": "stat"
1773     },
1774     {
1775       "datasource": {
1776         "type": "prometheus",
1777         "uid": "PBFA97CFB590B2093"
1778       },
1779       "description": "Offset Commit request rate.",
1780       "fieldConfig": {
1781         "defaults": {
1782           "mappings": [
1783             {
1784               "options": {
1785                 "match": "null",
1786                 "result": {
1787                   "text": "N/A"
1788                 }
1789               },
1790               "type": "special"
1791             }
1792           ],
1793           "thresholds": {
1794             "mode": "absolute",
1795             "steps": [
1796               {
1797                 "color": "green",
1798                 "value": null
1799               }
1800             ]
1801           },
1802           "unit": "none"
1803         },
1804         "overrides": []
1805       },
1806       "gridPos": {
1807         "h": 4,
1808         "w": 4,
1809         "x": 16,
1810         "y": 23
1811       },
1812       "id": 38,
1813       "links": [],
1814       "maxDataPoints": 100,
1815       "options": {
1816         "colorMode": "value",
1817         "graphMode": "area",
1818         "justifyMode": "auto",
1819         "orientation": "horizontal",
1820         "reduceOptions": {
1821           "calcs": [
1822             "lastNotNull"
1823           ],
1824           "fields": "",
1825           "values": false
1826         },
1827         "text": {},
1828         "textMode": "auto"
1829       },
1830       "pluginVersion": "10.0.3",
1831       "targets": [
1832         {
1833           "datasource": {
1834             "type": "prometheus",
1835             "uid": "PBFA97CFB590B2093"
1836           },
1837           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1838           "interval": "",
1839           "legendFormat": "",
1840           "refId": "A"
1841         }
1842       ],
1843       "title": "Offset Commit Request Per Sec",
1844       "type": "stat"
1845     },
1846     {
1847       "datasource": {
1848         "type": "prometheus",
1849         "uid": "PBFA97CFB590B2093"
1850       },
1851       "description": "Fetch request rate.",
1852       "fieldConfig": {
1853         "defaults": {
1854           "mappings": [
1855             {
1856               "options": {
1857                 "match": "null",
1858                 "result": {
1859                   "text": "N/A"
1860                 }
1861               },
1862               "type": "special"
1863             }
1864           ],
1865           "thresholds": {
1866             "mode": "absolute",
1867             "steps": [
1868               {
1869                 "color": "green",
1870                 "value": null
1871               }
1872             ]
1873           },
1874           "unit": "none"
1875         },
1876         "overrides": []
1877       },
1878       "gridPos": {
1879         "h": 4,
1880         "w": 4,
1881         "x": 20,
1882         "y": 23
1883       },
1884       "id": 94,
1885       "links": [],
1886       "maxDataPoints": 100,
1887       "options": {
1888         "colorMode": "value",
1889         "graphMode": "area",
1890         "justifyMode": "auto",
1891         "orientation": "horizontal",
1892         "reduceOptions": {
1893           "calcs": [
1894             "lastNotNull"
1895           ],
1896           "fields": "",
1897           "values": false
1898         },
1899         "text": {},
1900         "textMode": "auto"
1901       },
1902       "pluginVersion": "10.0.3",
1903       "targets": [
1904         {
1905           "datasource": {
1906             "type": "prometheus",
1907             "uid": "PBFA97CFB590B2093"
1908           },
1909           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1910           "interval": "",
1911           "legendFormat": "",
1912           "refId": "A"
1913         }
1914       ],
1915       "title": "Broker Fetch Request Per Sec",
1916       "type": "stat"
1917     },
1918     {
1919       "collapsed": false,
1920       "datasource": {
1921         "type": "datasource",
1922         "uid": "grafana"
1923       },
1924       "gridPos": {
1925         "h": 1,
1926         "w": 24,
1927         "x": 0,
1928         "y": 27
1929       },
1930       "id": 29,
1931       "panels": [],
1932       "targets": [
1933         {
1934           "datasource": {
1935             "type": "datasource",
1936             "uid": "grafana"
1937           },
1938           "refId": "A"
1939         }
1940       ],
1941       "title": "Throughput In/Out",
1942       "type": "row"
1943     },
1944     {
1945       "datasource": {
1946         "type": "prometheus",
1947         "uid": "PBFA97CFB590B2093"
1948       },
1949       "fieldConfig": {
1950         "defaults": {
1951           "color": {
1952             "mode": "palette-classic"
1953           },
1954           "custom": {
1955             "axisCenteredZero": false,
1956             "axisColorMode": "text",
1957             "axisLabel": "Messages/s",
1958             "axisPlacement": "auto",
1959             "barAlignment": 0,
1960             "drawStyle": "line",
1961             "fillOpacity": 10,
1962             "gradientMode": "none",
1963             "hideFrom": {
1964               "legend": false,
1965               "tooltip": false,
1966               "viz": false
1967             },
1968             "lineInterpolation": "linear",
1969             "lineWidth": 2,
1970             "pointSize": 5,
1971             "scaleDistribution": {
1972               "type": "linear"
1973             },
1974             "showPoints": "never",
1975             "spanNulls": false,
1976             "stacking": {
1977               "group": "A",
1978               "mode": "none"
1979             },
1980             "thresholdsStyle": {
1981               "mode": "off"
1982             }
1983           },
1984           "links": [],
1985           "mappings": [],
1986           "min": 0,
1987           "thresholds": {
1988             "mode": "absolute",
1989             "steps": [
1990               {
1991                 "color": "green",
1992                 "value": null
1993               }
1994             ]
1995           },
1996           "unit": "cps"
1997         },
1998         "overrides": []
1999       },
2000       "gridPos": {
2001         "h": 8,
2002         "w": 8,
2003         "x": 0,
2004         "y": 28
2005       },
2006       "id": 4,
2007       "links": [],
2008       "options": {
2009         "legend": {
2010           "calcs": [
2011             "mean",
2012             "lastNotNull",
2013             "max"
2014           ],
2015           "displayMode": "table",
2016           "placement": "bottom",
2017           "showLegend": true
2018         },
2019         "tooltip": {
2020           "mode": "single",
2021           "sort": "none"
2022         }
2023       },
2024       "pluginVersion": "8.1.3",
2025       "targets": [
2026         {
2027           "datasource": {
2028             "type": "prometheus",
2029             "uid": "PBFA97CFB590B2093"
2030           },
2031           "exemplar": true,
2032           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2033           "interval": "",
2034           "intervalFactor": 2,
2035           "legendFormat": "bytes/sec",
2036           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2037           "refId": "A",
2038           "step": 4
2039         }
2040       ],
2041       "title": "Messages In",
2042       "type": "timeseries"
2043     },
2044     {
2045       "datasource": {
2046         "type": "prometheus",
2047         "uid": "PBFA97CFB590B2093"
2048       },
2049       "fieldConfig": {
2050         "defaults": {
2051           "color": {
2052             "mode": "palette-classic"
2053           },
2054           "custom": {
2055             "axisCenteredZero": false,
2056             "axisColorMode": "text",
2057             "axisLabel": "Bytes/s",
2058             "axisPlacement": "auto",
2059             "barAlignment": 0,
2060             "drawStyle": "line",
2061             "fillOpacity": 10,
2062             "gradientMode": "none",
2063             "hideFrom": {
2064               "legend": false,
2065               "tooltip": false,
2066               "viz": false
2067             },
2068             "lineInterpolation": "linear",
2069             "lineWidth": 2,
2070             "pointSize": 5,
2071             "scaleDistribution": {
2072               "type": "linear"
2073             },
2074             "showPoints": "never",
2075             "spanNulls": false,
2076             "stacking": {
2077               "group": "A",
2078               "mode": "none"
2079             },
2080             "thresholdsStyle": {
2081               "mode": "off"
2082             }
2083           },
2084           "links": [],
2085           "mappings": [],
2086           "min": 0,
2087           "thresholds": {
2088             "mode": "absolute",
2089             "steps": [
2090               {
2091                 "color": "green",
2092                 "value": null
2093               },
2094               {
2095                 "color": "red",
2096                 "value": 80
2097               }
2098             ]
2099           },
2100           "unit": "Bps"
2101         },
2102         "overrides": []
2103       },
2104       "gridPos": {
2105         "h": 8,
2106         "w": 8,
2107         "x": 8,
2108         "y": 28
2109       },
2110       "id": 5,
2111       "links": [],
2112       "options": {
2113         "legend": {
2114           "calcs": [
2115             "mean",
2116             "lastNotNull",
2117             "max"
2118           ],
2119           "displayMode": "table",
2120           "placement": "bottom",
2121           "showLegend": true
2122         },
2123         "tooltip": {
2124           "mode": "single",
2125           "sort": "none"
2126         }
2127       },
2128       "pluginVersion": "8.1.3",
2129       "targets": [
2130         {
2131           "datasource": {
2132             "type": "prometheus",
2133             "uid": "PBFA97CFB590B2093"
2134           },
2135           "exemplar": true,
2136           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2137           "format": "time_series",
2138           "interval": "",
2139           "intervalFactor": 2,
2140           "legendFormat": "bytes/sec",
2141           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2142           "refId": "A",
2143           "step": 4
2144         }
2145       ],
2146       "title": "Bytes In",
2147       "type": "timeseries"
2148     },
2149     {
2150       "datasource": {
2151         "type": "prometheus",
2152         "uid": "PBFA97CFB590B2093"
2153       },
2154       "fieldConfig": {
2155         "defaults": {
2156           "color": {
2157             "mode": "palette-classic"
2158           },
2159           "custom": {
2160             "axisCenteredZero": false,
2161             "axisColorMode": "text",
2162             "axisLabel": "Bytes/s",
2163             "axisPlacement": "auto",
2164             "barAlignment": 0,
2165             "drawStyle": "line",
2166             "fillOpacity": 10,
2167             "gradientMode": "none",
2168             "hideFrom": {
2169               "legend": false,
2170               "tooltip": false,
2171               "viz": false
2172             },
2173             "lineInterpolation": "linear",
2174             "lineWidth": 2,
2175             "pointSize": 5,
2176             "scaleDistribution": {
2177               "type": "linear"
2178             },
2179             "showPoints": "never",
2180             "spanNulls": false,
2181             "stacking": {
2182               "group": "A",
2183               "mode": "none"
2184             },
2185             "thresholdsStyle": {
2186               "mode": "off"
2187             }
2188           },
2189           "links": [],
2190           "mappings": [],
2191           "min": 0,
2192           "thresholds": {
2193             "mode": "absolute",
2194             "steps": [
2195               {
2196                 "color": "green",
2197                 "value": null
2198               },
2199               {
2200                 "color": "red",
2201                 "value": 80
2202               }
2203             ]
2204           },
2205           "unit": "Bps"
2206         },
2207         "overrides": []
2208       },
2209       "gridPos": {
2210         "h": 8,
2211         "w": 8,
2212         "x": 16,
2213         "y": 28
2214       },
2215       "id": 6,
2216       "links": [],
2217       "options": {
2218         "legend": {
2219           "calcs": [
2220             "mean",
2221             "lastNotNull",
2222             "max"
2223           ],
2224           "displayMode": "table",
2225           "placement": "bottom",
2226           "showLegend": true
2227         },
2228         "tooltip": {
2229           "mode": "single",
2230           "sort": "none"
2231         }
2232       },
2233       "pluginVersion": "8.1.3",
2234       "targets": [
2235         {
2236           "datasource": {
2237             "type": "prometheus",
2238             "uid": "PBFA97CFB590B2093"
2239           },
2240           "exemplar": true,
2241           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2242           "interval": "",
2243           "intervalFactor": 2,
2244           "legendFormat": "bytes/sec",
2245           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2246           "refId": "A",
2247           "step": 4
2248         }
2249       ],
2250       "title": "Bytes Out",
2251       "type": "timeseries"
2252     },
2253     {
2254       "datasource": {
2255         "type": "prometheus",
2256         "uid": "PBFA97CFB590B2093"
2257       },
2258       "fieldConfig": {
2259         "defaults": {
2260           "color": {
2261             "mode": "palette-classic"
2262           },
2263           "custom": {
2264             "axisCenteredZero": false,
2265             "axisColorMode": "text",
2266             "axisLabel": "Messages/s",
2267             "axisPlacement": "auto",
2268             "barAlignment": 0,
2269             "drawStyle": "line",
2270             "fillOpacity": 10,
2271             "gradientMode": "none",
2272             "hideFrom": {
2273               "legend": false,
2274               "tooltip": false,
2275               "viz": false
2276             },
2277             "lineInterpolation": "linear",
2278             "lineWidth": 2,
2279             "pointSize": 5,
2280             "scaleDistribution": {
2281               "type": "linear"
2282             },
2283             "showPoints": "never",
2284             "spanNulls": false,
2285             "stacking": {
2286               "group": "A",
2287               "mode": "none"
2288             },
2289             "thresholdsStyle": {
2290               "mode": "off"
2291             }
2292           },
2293           "links": [],
2294           "mappings": [],
2295           "thresholds": {
2296             "mode": "absolute",
2297             "steps": [
2298               {
2299                 "color": "green",
2300                 "value": null
2301               },
2302               {
2303                 "color": "red",
2304                 "value": 80
2305               }
2306             ]
2307           },
2308           "unit": "cps"
2309         },
2310         "overrides": []
2311       },
2312       "gridPos": {
2313         "h": 8,
2314         "w": 8,
2315         "x": 0,
2316         "y": 36
2317       },
2318       "id": 10,
2319       "links": [],
2320       "options": {
2321         "legend": {
2322           "calcs": [
2323             "mean",
2324             "lastNotNull",
2325             "max"
2326           ],
2327           "displayMode": "table",
2328           "placement": "bottom",
2329           "showLegend": true
2330         },
2331         "tooltip": {
2332           "mode": "single",
2333           "sort": "none"
2334         }
2335       },
2336       "pluginVersion": "8.1.3",
2337       "targets": [
2338         {
2339           "datasource": {
2340             "type": "prometheus",
2341             "uid": "PBFA97CFB590B2093"
2342           },
2343           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2344           "format": "time_series",
2345           "interval": "",
2346           "intervalFactor": 2,
2347           "legendFormat": "{{instance}}",
2348           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2349           "refId": "A",
2350           "step": 4
2351         }
2352       ],
2353       "title": "Messages In Per Broker",
2354       "type": "timeseries"
2355     },
2356     {
2357       "datasource": {
2358         "type": "prometheus",
2359         "uid": "PBFA97CFB590B2093"
2360       },
2361       "fieldConfig": {
2362         "defaults": {
2363           "color": {
2364             "mode": "palette-classic"
2365           },
2366           "custom": {
2367             "axisCenteredZero": false,
2368             "axisColorMode": "text",
2369             "axisLabel": "Bytes/s",
2370             "axisPlacement": "auto",
2371             "barAlignment": 0,
2372             "drawStyle": "line",
2373             "fillOpacity": 10,
2374             "gradientMode": "none",
2375             "hideFrom": {
2376               "legend": false,
2377               "tooltip": false,
2378               "viz": false
2379             },
2380             "lineInterpolation": "linear",
2381             "lineWidth": 2,
2382             "pointSize": 5,
2383             "scaleDistribution": {
2384               "type": "linear"
2385             },
2386             "showPoints": "never",
2387             "spanNulls": false,
2388             "stacking": {
2389               "group": "A",
2390               "mode": "none"
2391             },
2392             "thresholdsStyle": {
2393               "mode": "off"
2394             }
2395           },
2396           "links": [],
2397           "mappings": [],
2398           "thresholds": {
2399             "mode": "absolute",
2400             "steps": [
2401               {
2402                 "color": "green",
2403                 "value": null
2404               },
2405               {
2406                 "color": "red",
2407                 "value": 80
2408               }
2409             ]
2410           },
2411           "unit": "Bps"
2412         },
2413         "overrides": []
2414       },
2415       "gridPos": {
2416         "h": 8,
2417         "w": 8,
2418         "x": 8,
2419         "y": 36
2420       },
2421       "id": 7,
2422       "links": [],
2423       "options": {
2424         "legend": {
2425           "calcs": [
2426             "mean",
2427             "lastNotNull",
2428             "max"
2429           ],
2430           "displayMode": "table",
2431           "placement": "bottom",
2432           "showLegend": true
2433         },
2434         "tooltip": {
2435           "mode": "single",
2436           "sort": "none"
2437         }
2438       },
2439       "pluginVersion": "8.1.3",
2440       "targets": [
2441         {
2442           "datasource": {
2443             "type": "prometheus",
2444             "uid": "PBFA97CFB590B2093"
2445           },
2446           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2447           "format": "time_series",
2448           "interval": "",
2449           "intervalFactor": 2,
2450           "legendFormat": "{{instance}}",
2451           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2452           "refId": "A",
2453           "step": 4
2454         }
2455       ],
2456       "title": "Bytes In Per Broker",
2457       "type": "timeseries"
2458     },
2459     {
2460       "datasource": {
2461         "type": "prometheus",
2462         "uid": "PBFA97CFB590B2093"
2463       },
2464       "fieldConfig": {
2465         "defaults": {
2466           "color": {
2467             "mode": "palette-classic"
2468           },
2469           "custom": {
2470             "axisCenteredZero": false,
2471             "axisColorMode": "text",
2472             "axisLabel": "",
2473             "axisPlacement": "auto",
2474             "barAlignment": 0,
2475             "drawStyle": "line",
2476             "fillOpacity": 10,
2477             "gradientMode": "none",
2478             "hideFrom": {
2479               "legend": false,
2480               "tooltip": false,
2481               "viz": false
2482             },
2483             "lineInterpolation": "linear",
2484             "lineWidth": 1,
2485             "pointSize": 5,
2486             "scaleDistribution": {
2487               "type": "linear"
2488             },
2489             "showPoints": "never",
2490             "spanNulls": true,
2491             "stacking": {
2492               "group": "A",
2493               "mode": "none"
2494             },
2495             "thresholdsStyle": {
2496               "mode": "off"
2497             }
2498           },
2499           "links": [],
2500           "mappings": [],
2501           "min": 0,
2502           "thresholds": {
2503             "mode": "absolute",
2504             "steps": [
2505               {
2506                 "color": "green",
2507                 "value": null
2508               },
2509               {
2510                 "color": "red",
2511                 "value": 80
2512               }
2513             ]
2514           },
2515           "unit": "binBps"
2516         },
2517         "overrides": []
2518       },
2519       "gridPos": {
2520         "h": 8,
2521         "w": 8,
2522         "x": 16,
2523         "y": 36
2524       },
2525       "id": 9,
2526       "links": [],
2527       "options": {
2528         "legend": {
2529           "calcs": [
2530             "mean",
2531             "lastNotNull",
2532             "max"
2533           ],
2534           "displayMode": "table",
2535           "placement": "bottom",
2536           "showLegend": true
2537         },
2538         "tooltip": {
2539           "mode": "single",
2540           "sort": "none"
2541         }
2542       },
2543       "pluginVersion": "8.1.3",
2544       "targets": [
2545         {
2546           "datasource": {
2547             "type": "prometheus",
2548             "uid": "PBFA97CFB590B2093"
2549           },
2550           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2551           "format": "time_series",
2552           "interval": "",
2553           "intervalFactor": 1,
2554           "legendFormat": "{{instance}}",
2555           "refId": "A"
2556         }
2557       ],
2558       "title": "Bytes Out Per Broker",
2559       "type": "timeseries"
2560     },
2561     {
2562       "collapsed": false,
2563       "datasource": {
2564         "type": "datasource",
2565         "uid": "grafana"
2566       },
2567       "gridPos": {
2568         "h": 1,
2569         "w": 24,
2570         "x": 0,
2571         "y": 44
2572       },
2573       "id": 44,
2574       "panels": [],
2575       "targets": [
2576         {
2577           "datasource": {
2578             "type": "datasource",
2579             "uid": "grafana"
2580           },
2581           "refId": "A"
2582         }
2583       ],
2584       "title": "Thread utilization",
2585       "type": "row"
2586     },
2587     {
2588       "datasource": {
2589         "type": "prometheus",
2590         "uid": "PBFA97CFB590B2093"
2591       },
2592       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2593       "fieldConfig": {
2594         "defaults": {
2595           "color": {
2596             "mode": "palette-classic"
2597           },
2598           "custom": {
2599             "axisCenteredZero": false,
2600             "axisColorMode": "text",
2601             "axisLabel": "",
2602             "axisPlacement": "auto",
2603             "barAlignment": 0,
2604             "drawStyle": "line",
2605             "fillOpacity": 10,
2606             "gradientMode": "none",
2607             "hideFrom": {
2608               "legend": false,
2609               "tooltip": false,
2610               "viz": false
2611             },
2612             "lineInterpolation": "linear",
2613             "lineWidth": 1,
2614             "pointSize": 5,
2615             "scaleDistribution": {
2616               "type": "linear"
2617             },
2618             "showPoints": "never",
2619             "spanNulls": true,
2620             "stacking": {
2621               "group": "A",
2622               "mode": "none"
2623             },
2624             "thresholdsStyle": {
2625               "mode": "off"
2626             }
2627           },
2628           "links": [],
2629           "mappings": [],
2630           "thresholds": {
2631             "mode": "absolute",
2632             "steps": [
2633               {
2634                 "color": "green",
2635                 "value": null
2636               },
2637               {
2638                 "color": "red",
2639                 "value": 80
2640               }
2641             ]
2642           },
2643           "unit": "percentunit"
2644         },
2645         "overrides": []
2646       },
2647       "gridPos": {
2648         "h": 12,
2649         "w": 12,
2650         "x": 0,
2651         "y": 45
2652       },
2653       "id": 24,
2654       "options": {
2655         "legend": {
2656           "calcs": [
2657             "mean",
2658             "lastNotNull",
2659             "max"
2660           ],
2661           "displayMode": "table",
2662           "placement": "bottom",
2663           "showLegend": true
2664         },
2665         "tooltip": {
2666           "mode": "single",
2667           "sort": "none"
2668         }
2669       },
2670       "pluginVersion": "8.1.3",
2671       "targets": [
2672         {
2673           "datasource": {
2674             "type": "prometheus",
2675             "uid": "PBFA97CFB590B2093"
2676           },
2677           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2678           "interval": "",
2679           "legendFormat": "{{instance}}",
2680           "refId": "A"
2681         }
2682       ],
2683       "title": "Network Processor Avg Usage Percent",
2684       "type": "timeseries"
2685     },
2686     {
2687       "datasource": {
2688         "type": "prometheus",
2689         "uid": "PBFA97CFB590B2093"
2690       },
2691       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2692       "fieldConfig": {
2693         "defaults": {
2694           "color": {
2695             "mode": "palette-classic"
2696           },
2697           "custom": {
2698             "axisCenteredZero": false,
2699             "axisColorMode": "text",
2700             "axisLabel": "",
2701             "axisPlacement": "auto",
2702             "barAlignment": 0,
2703             "drawStyle": "line",
2704             "fillOpacity": 10,
2705             "gradientMode": "none",
2706             "hideFrom": {
2707               "legend": false,
2708               "tooltip": false,
2709               "viz": false
2710             },
2711             "lineInterpolation": "linear",
2712             "lineWidth": 1,
2713             "pointSize": 5,
2714             "scaleDistribution": {
2715               "type": "linear"
2716             },
2717             "showPoints": "never",
2718             "spanNulls": true,
2719             "stacking": {
2720               "group": "A",
2721               "mode": "none"
2722             },
2723             "thresholdsStyle": {
2724               "mode": "off"
2725             }
2726           },
2727           "links": [],
2728           "mappings": [],
2729           "min": 0,
2730           "thresholds": {
2731             "mode": "absolute",
2732             "steps": [
2733               {
2734                 "color": "green",
2735                 "value": null
2736               },
2737               {
2738                 "color": "red",
2739                 "value": 80
2740               }
2741             ]
2742           },
2743           "unit": "percentunit"
2744         },
2745         "overrides": []
2746       },
2747       "gridPos": {
2748         "h": 12,
2749         "w": 12,
2750         "x": 12,
2751         "y": 45
2752       },
2753       "id": 25,
2754       "options": {
2755         "legend": {
2756           "calcs": [
2757             "mean",
2758             "lastNotNull",
2759             "max"
2760           ],
2761           "displayMode": "table",
2762           "placement": "bottom",
2763           "showLegend": true
2764         },
2765         "tooltip": {
2766           "mode": "single",
2767           "sort": "none"
2768         }
2769       },
2770       "pluginVersion": "8.1.3",
2771       "targets": [
2772         {
2773           "datasource": {
2774             "type": "prometheus",
2775             "uid": "PBFA97CFB590B2093"
2776           },
2777           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2778           "interval": "",
2779           "legendFormat": "{{instance}}",
2780           "refId": "A"
2781         }
2782       ],
2783       "title": "Request Handler Avg Percent",
2784       "type": "timeseries"
2785     },
2786     {
2787       "collapsed": false,
2788       "datasource": {
2789         "type": "datasource",
2790         "uid": "grafana"
2791       },
2792       "gridPos": {
2793         "h": 1,
2794         "w": 24,
2795         "x": 0,
2796         "y": 57
2797       },
2798       "id": 82,
2799       "panels": [],
2800       "targets": [
2801         {
2802           "datasource": {
2803             "type": "datasource",
2804             "uid": "grafana"
2805           },
2806           "refId": "A"
2807         }
2808       ],
2809       "title": "Isr Shrinks / Expands",
2810       "type": "row"
2811     },
2812     {
2813       "datasource": {
2814         "type": "prometheus",
2815         "uid": "PBFA97CFB590B2093"
2816       },
2817       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2818       "fieldConfig": {
2819         "defaults": {
2820           "color": {
2821             "mode": "palette-classic"
2822           },
2823           "custom": {
2824             "axisCenteredZero": false,
2825             "axisColorMode": "text",
2826             "axisLabel": "",
2827             "axisPlacement": "auto",
2828             "barAlignment": 0,
2829             "drawStyle": "line",
2830             "fillOpacity": 10,
2831             "gradientMode": "none",
2832             "hideFrom": {
2833               "legend": false,
2834               "tooltip": false,
2835               "viz": false
2836             },
2837             "lineInterpolation": "linear",
2838             "lineWidth": 1,
2839             "pointSize": 5,
2840             "scaleDistribution": {
2841               "type": "linear"
2842             },
2843             "showPoints": "never",
2844             "spanNulls": true,
2845             "stacking": {
2846               "group": "A",
2847               "mode": "none"
2848             },
2849             "thresholdsStyle": {
2850               "mode": "off"
2851             }
2852           },
2853           "links": [],
2854           "mappings": [],
2855           "min": 0,
2856           "thresholds": {
2857             "mode": "absolute",
2858             "steps": [
2859               {
2860                 "color": "green"
2861               },
2862               {
2863                 "color": "red",
2864                 "value": 80
2865               }
2866             ]
2867           },
2868           "unit": "ops"
2869         },
2870         "overrides": [
2871           {
2872             "__systemRef": "hideSeriesFrom",
2873             "matcher": {
2874               "id": "byNames",
2875               "options": {
2876                 "mode": "exclude",
2877                 "names": [
2878                   "kafka-2:80"
2879                 ],
2880                 "prefix": "All except:",
2881                 "readOnly": true
2882               }
2883             },
2884             "properties": [
2885               {
2886                 "id": "custom.hideFrom",
2887                 "value": {
2888                   "legend": false,
2889                   "tooltip": false,
2890                   "viz": true
2891                 }
2892               }
2893             ]
2894           }
2895         ]
2896       },
2897       "gridPos": {
2898         "h": 9,
2899         "w": 12,
2900         "x": 0,
2901         "y": 58
2902       },
2903       "id": 80,
2904       "options": {
2905         "legend": {
2906           "calcs": [
2907             "mean",
2908             "lastNotNull",
2909             "max"
2910           ],
2911           "displayMode": "table",
2912           "placement": "bottom",
2913           "showLegend": true
2914         },
2915         "tooltip": {
2916           "mode": "single",
2917           "sort": "none"
2918         }
2919       },
2920       "pluginVersion": "8.1.3",
2921       "targets": [
2922         {
2923           "datasource": {
2924             "type": "prometheus",
2925             "uid": "PBFA97CFB590B2093"
2926           },
2927           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2928           "interval": "",
2929           "legendFormat": "{{instance}}",
2930           "refId": "A"
2931         }
2932       ],
2933       "title": "IsrShrinks per Sec",
2934       "type": "timeseries"
2935     },
2936     {
2937       "datasource": {
2938         "type": "prometheus",
2939         "uid": "PBFA97CFB590B2093"
2940       },
2941       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2942       "fieldConfig": {
2943         "defaults": {
2944           "color": {
2945             "mode": "palette-classic"
2946           },
2947           "custom": {
2948             "axisCenteredZero": false,
2949             "axisColorMode": "text",
2950             "axisLabel": "",
2951             "axisPlacement": "auto",
2952             "barAlignment": 0,
2953             "drawStyle": "line",
2954             "fillOpacity": 10,
2955             "gradientMode": "none",
2956             "hideFrom": {
2957               "legend": false,
2958               "tooltip": false,
2959               "viz": false
2960             },
2961             "lineInterpolation": "linear",
2962             "lineWidth": 1,
2963             "pointSize": 5,
2964             "scaleDistribution": {
2965               "type": "linear"
2966             },
2967             "showPoints": "never",
2968             "spanNulls": true,
2969             "stacking": {
2970               "group": "A",
2971               "mode": "none"
2972             },
2973             "thresholdsStyle": {
2974               "mode": "off"
2975             }
2976           },
2977           "links": [],
2978           "mappings": [],
2979           "min": 0,
2980           "thresholds": {
2981             "mode": "absolute",
2982             "steps": [
2983               {
2984                 "color": "green"
2985               },
2986               {
2987                 "color": "red",
2988                 "value": 80
2989               }
2990             ]
2991           },
2992           "unit": "ops"
2993         },
2994         "overrides": [
2995           {
2996             "__systemRef": "hideSeriesFrom",
2997             "matcher": {
2998               "id": "byNames",
2999               "options": {
3000                 "mode": "exclude",
3001                 "names": [
3002                   "kafka-2:80"
3003                 ],
3004                 "prefix": "All except:",
3005                 "readOnly": true
3006               }
3007             },
3008             "properties": [
3009               {
3010                 "id": "custom.hideFrom",
3011                 "value": {
3012                   "legend": false,
3013                   "tooltip": false,
3014                   "viz": true
3015                 }
3016               }
3017             ]
3018           }
3019         ]
3020       },
3021       "gridPos": {
3022         "h": 9,
3023         "w": 12,
3024         "x": 12,
3025         "y": 58
3026       },
3027       "id": 83,
3028       "options": {
3029         "legend": {
3030           "calcs": [
3031             "mean",
3032             "lastNotNull",
3033             "max"
3034           ],
3035           "displayMode": "table",
3036           "placement": "bottom",
3037           "showLegend": true
3038         },
3039         "tooltip": {
3040           "mode": "single",
3041           "sort": "none"
3042         }
3043       },
3044       "pluginVersion": "8.1.3",
3045       "targets": [
3046         {
3047           "datasource": {
3048             "type": "prometheus",
3049             "uid": "PBFA97CFB590B2093"
3050           },
3051           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3052           "hide": false,
3053           "interval": "",
3054           "legendFormat": "{{instance}}",
3055           "refId": "A"
3056         }
3057       ],
3058       "title": "IsrExpands per Sec",
3059       "type": "timeseries"
3060     },
3061     {
3062       "collapsed": true,
3063       "datasource": {
3064         "type": "datasource",
3065         "uid": "grafana"
3066       },
3067       "gridPos": {
3068         "h": 1,
3069         "w": 24,
3070         "x": 0,
3071         "y": 67
3072       },
3073       "id": 53,
3074       "panels": [
3075         {
3076           "datasource": {
3077             "type": "prometheus",
3078             "uid": "PBFA97CFB590B2093"
3079           },
3080           "fieldConfig": {
3081             "defaults": {
3082               "color": {
3083                 "mode": "palette-classic"
3084               },
3085               "custom": {
3086                 "axisLabel": "",
3087                 "axisPlacement": "auto",
3088                 "barAlignment": 0,
3089                 "drawStyle": "line",
3090                 "fillOpacity": 10,
3091                 "gradientMode": "none",
3092                 "hideFrom": {
3093                   "legend": false,
3094                   "tooltip": false,
3095                   "viz": false
3096                 },
3097                 "lineInterpolation": "linear",
3098                 "lineWidth": 1,
3099                 "pointSize": 5,
3100                 "scaleDistribution": {
3101                   "type": "linear"
3102                 },
3103                 "showPoints": "never",
3104                 "spanNulls": true,
3105                 "stacking": {
3106                   "group": "A",
3107                   "mode": "none"
3108                 },
3109                 "thresholdsStyle": {
3110                   "mode": "off"
3111                 }
3112               },
3113               "links": [],
3114               "mappings": [],
3115               "thresholds": {
3116                 "mode": "absolute",
3117                 "steps": [
3118                   {
3119                     "color": "green"
3120                   },
3121                   {
3122                     "color": "red",
3123                     "value": 80
3124                   }
3125                 ]
3126               },
3127               "unit": "bytes"
3128             },
3129             "overrides": []
3130           },
3131           "gridPos": {
3132             "h": 9,
3133             "w": 12,
3134             "x": 0,
3135             "y": 28
3136           },
3137           "id": 55,
3138           "options": {
3139             "legend": {
3140               "calcs": [
3141                 "mean",
3142                 "lastNotNull",
3143                 "max"
3144               ],
3145               "displayMode": "table",
3146               "placement": "bottom",
3147               "showLegend": true
3148             },
3149             "tooltip": {
3150               "mode": "single"
3151             }
3152           },
3153           "pluginVersion": "8.1.3",
3154           "targets": [
3155             {
3156               "datasource": {
3157                 "type": "prometheus",
3158                 "uid": "PBFA97CFB590B2093"
3159               },
3160               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3161               "legendFormat": "{{topic}}",
3162               "refId": "A"
3163             }
3164           ],
3165           "title": "Log size per Topic",
3166           "type": "timeseries"
3167         },
3168         {
3169           "datasource": {
3170             "type": "prometheus",
3171             "uid": "PBFA97CFB590B2093"
3172           },
3173           "fieldConfig": {
3174             "defaults": {
3175               "color": {
3176                 "mode": "palette-classic"
3177               },
3178               "custom": {
3179                 "axisLabel": "",
3180                 "axisPlacement": "auto",
3181                 "barAlignment": 0,
3182                 "drawStyle": "line",
3183                 "fillOpacity": 10,
3184                 "gradientMode": "none",
3185                 "hideFrom": {
3186                   "legend": false,
3187                   "tooltip": false,
3188                   "viz": false
3189                 },
3190                 "lineInterpolation": "linear",
3191                 "lineWidth": 1,
3192                 "pointSize": 5,
3193                 "scaleDistribution": {
3194                   "type": "linear"
3195                 },
3196                 "showPoints": "never",
3197                 "spanNulls": true,
3198                 "stacking": {
3199                   "group": "A",
3200                   "mode": "none"
3201                 },
3202                 "thresholdsStyle": {
3203                   "mode": "off"
3204                 }
3205               },
3206               "links": [],
3207               "mappings": [],
3208               "thresholds": {
3209                 "mode": "absolute",
3210                 "steps": [
3211                   {
3212                     "color": "green"
3213                   },
3214                   {
3215                     "color": "red",
3216                     "value": 80
3217                   }
3218                 ]
3219               },
3220               "unit": "bytes"
3221             },
3222             "overrides": []
3223           },
3224           "gridPos": {
3225             "h": 9,
3226             "w": 12,
3227             "x": 12,
3228             "y": 28
3229           },
3230           "id": 56,
3231           "options": {
3232             "legend": {
3233               "calcs": [
3234                 "mean",
3235                 "lastNotNull",
3236                 "max"
3237               ],
3238               "displayMode": "table",
3239               "placement": "bottom",
3240               "showLegend": true
3241             },
3242             "tooltip": {
3243               "mode": "single"
3244             }
3245           },
3246           "pluginVersion": "8.1.3",
3247           "targets": [
3248             {
3249               "datasource": {
3250                 "type": "prometheus",
3251                 "uid": "PBFA97CFB590B2093"
3252               },
3253               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3254               "legendFormat": "{{instance}}",
3255               "refId": "A"
3256             }
3257           ],
3258           "title": "Log size per Broker",
3259           "type": "timeseries"
3260         }
3261       ],
3262       "targets": [
3263         {
3264           "datasource": {
3265             "type": "datasource",
3266             "uid": "grafana"
3267           },
3268           "refId": "A"
3269         }
3270       ],
3271       "title": "Logs size",
3272       "type": "row"
3273     },
3274     {
3275       "collapsed": true,
3276       "datasource": {
3277         "type": "datasource",
3278         "uid": "grafana"
3279       },
3280       "gridPos": {
3281         "h": 1,
3282         "w": 24,
3283         "x": 0,
3284         "y": 68
3285       },
3286       "id": 58,
3287       "panels": [
3288         {
3289           "datasource": {
3290             "type": "prometheus",
3291             "uid": "PBFA97CFB590B2093"
3292           },
3293           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3294           "fieldConfig": {
3295             "defaults": {
3296               "color": {
3297                 "mode": "palette-classic"
3298               },
3299               "custom": {
3300                 "axisLabel": "",
3301                 "axisPlacement": "auto",
3302                 "barAlignment": 0,
3303                 "drawStyle": "line",
3304                 "fillOpacity": 10,
3305                 "gradientMode": "none",
3306                 "hideFrom": {
3307                   "legend": false,
3308                   "tooltip": false,
3309                   "viz": false
3310                 },
3311                 "lineInterpolation": "linear",
3312                 "lineWidth": 1,
3313                 "pointSize": 5,
3314                 "scaleDistribution": {
3315                   "type": "linear"
3316                 },
3317                 "showPoints": "never",
3318                 "spanNulls": true,
3319                 "stacking": {
3320                   "group": "A",
3321                   "mode": "none"
3322                 },
3323                 "thresholdsStyle": {
3324                   "mode": "off"
3325                 }
3326               },
3327               "links": [],
3328               "mappings": [],
3329               "thresholds": {
3330                 "mode": "absolute",
3331                 "steps": [
3332                   {
3333                     "color": "green"
3334                   },
3335                   {
3336                     "color": "red",
3337                     "value": 80
3338                   }
3339                 ]
3340               },
3341               "unit": "ms"
3342             },
3343             "overrides": []
3344           },
3345           "gridPos": {
3346             "h": 9,
3347             "w": 12,
3348             "x": 0,
3349             "y": 29
3350           },
3351           "id": 60,
3352           "options": {
3353             "legend": {
3354               "calcs": [
3355                 "mean",
3356                 "lastNotNull",
3357                 "max"
3358               ],
3359               "displayMode": "list",
3360               "placement": "bottom",
3361               "showLegend": true
3362             },
3363             "tooltip": {
3364               "mode": "single"
3365             }
3366           },
3367           "pluginVersion": "8.1.3",
3368           "targets": [
3369             {
3370               "datasource": {
3371                 "type": "prometheus",
3372                 "uid": "PBFA97CFB590B2093"
3373               },
3374               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3375               "hide": false,
3376               "legendFormat": "{{instance}} - {{quantile}}",
3377               "refId": "A"
3378             }
3379           ],
3380           "title": "Producer - RequestQueueTimeMs",
3381           "type": "timeseries"
3382         },
3383         {
3384           "datasource": {
3385             "type": "prometheus",
3386             "uid": "PBFA97CFB590B2093"
3387           },
3388           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3389           "fieldConfig": {
3390             "defaults": {
3391               "color": {
3392                 "mode": "palette-classic"
3393               },
3394               "custom": {
3395                 "axisLabel": "",
3396                 "axisPlacement": "auto",
3397                 "barAlignment": 0,
3398                 "drawStyle": "line",
3399                 "fillOpacity": 10,
3400                 "gradientMode": "none",
3401                 "hideFrom": {
3402                   "legend": false,
3403                   "tooltip": false,
3404                   "viz": false
3405                 },
3406                 "lineInterpolation": "linear",
3407                 "lineWidth": 1,
3408                 "pointSize": 5,
3409                 "scaleDistribution": {
3410                   "type": "linear"
3411                 },
3412                 "showPoints": "never",
3413                 "spanNulls": true,
3414                 "stacking": {
3415                   "group": "A",
3416                   "mode": "none"
3417                 },
3418                 "thresholdsStyle": {
3419                   "mode": "off"
3420                 }
3421               },
3422               "links": [],
3423               "mappings": [],
3424               "thresholds": {
3425                 "mode": "absolute",
3426                 "steps": [
3427                   {
3428                     "color": "green"
3429                   },
3430                   {
3431                     "color": "red",
3432                     "value": 80
3433                   }
3434                 ]
3435               },
3436               "unit": "ms"
3437             },
3438             "overrides": []
3439           },
3440           "gridPos": {
3441             "h": 9,
3442             "w": 12,
3443             "x": 12,
3444             "y": 29
3445           },
3446           "id": 61,
3447           "options": {
3448             "legend": {
3449               "calcs": [
3450                 "mean",
3451                 "lastNotNull",
3452                 "max"
3453               ],
3454               "displayMode": "list",
3455               "placement": "bottom",
3456               "showLegend": true
3457             },
3458             "tooltip": {
3459               "mode": "single"
3460             }
3461           },
3462           "pluginVersion": "8.1.3",
3463           "targets": [
3464             {
3465               "datasource": {
3466                 "type": "prometheus",
3467                 "uid": "PBFA97CFB590B2093"
3468               },
3469               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3470               "hide": false,
3471               "legendFormat": "{{instance}} - {{quantile}}",
3472               "refId": "A"
3473             }
3474           ],
3475           "title": "Producer - LocalTimeMs",
3476           "type": "timeseries"
3477         },
3478         {
3479           "datasource": {
3480             "type": "prometheus",
3481             "uid": "PBFA97CFB590B2093"
3482           },
3483           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3484           "fieldConfig": {
3485             "defaults": {
3486               "color": {
3487                 "mode": "palette-classic"
3488               },
3489               "custom": {
3490                 "axisLabel": "",
3491                 "axisPlacement": "auto",
3492                 "barAlignment": 0,
3493                 "drawStyle": "line",
3494                 "fillOpacity": 10,
3495                 "gradientMode": "none",
3496                 "hideFrom": {
3497                   "legend": false,
3498                   "tooltip": false,
3499                   "viz": false
3500                 },
3501                 "lineInterpolation": "linear",
3502                 "lineWidth": 1,
3503                 "pointSize": 5,
3504                 "scaleDistribution": {
3505                   "type": "linear"
3506                 },
3507                 "showPoints": "never",
3508                 "spanNulls": true,
3509                 "stacking": {
3510                   "group": "A",
3511                   "mode": "none"
3512                 },
3513                 "thresholdsStyle": {
3514                   "mode": "off"
3515                 }
3516               },
3517               "links": [],
3518               "mappings": [],
3519               "thresholds": {
3520                 "mode": "absolute",
3521                 "steps": [
3522                   {
3523                     "color": "green"
3524                   },
3525                   {
3526                     "color": "red",
3527                     "value": 80
3528                   }
3529                 ]
3530               },
3531               "unit": "ms"
3532             },
3533             "overrides": []
3534           },
3535           "gridPos": {
3536             "h": 10,
3537             "w": 8,
3538             "x": 0,
3539             "y": 38
3540           },
3541           "id": 62,
3542           "options": {
3543             "legend": {
3544               "calcs": [
3545                 "mean",
3546                 "lastNotNull",
3547                 "max"
3548               ],
3549               "displayMode": "table",
3550               "placement": "bottom",
3551               "showLegend": true
3552             },
3553             "tooltip": {
3554               "mode": "single"
3555             }
3556           },
3557           "pluginVersion": "8.1.3",
3558           "targets": [
3559             {
3560               "datasource": {
3561                 "type": "prometheus",
3562                 "uid": "PBFA97CFB590B2093"
3563               },
3564               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3565               "hide": false,
3566               "legendFormat": "{{instance}} - {{quantile}}",
3567               "refId": "A"
3568             }
3569           ],
3570           "title": "Producer - RemoteTimeMs",
3571           "type": "timeseries"
3572         },
3573         {
3574           "datasource": {
3575             "type": "prometheus",
3576             "uid": "PBFA97CFB590B2093"
3577           },
3578           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3579           "fieldConfig": {
3580             "defaults": {
3581               "color": {
3582                 "mode": "palette-classic"
3583               },
3584               "custom": {
3585                 "axisLabel": "",
3586                 "axisPlacement": "auto",
3587                 "barAlignment": 0,
3588                 "drawStyle": "line",
3589                 "fillOpacity": 10,
3590                 "gradientMode": "none",
3591                 "hideFrom": {
3592                   "legend": false,
3593                   "tooltip": false,
3594                   "viz": false
3595                 },
3596                 "lineInterpolation": "linear",
3597                 "lineWidth": 1,
3598                 "pointSize": 5,
3599                 "scaleDistribution": {
3600                   "type": "linear"
3601                 },
3602                 "showPoints": "never",
3603                 "spanNulls": true,
3604                 "stacking": {
3605                   "group": "A",
3606                   "mode": "none"
3607                 },
3608                 "thresholdsStyle": {
3609                   "mode": "off"
3610                 }
3611               },
3612               "links": [],
3613               "mappings": [],
3614               "thresholds": {
3615                 "mode": "absolute",
3616                 "steps": [
3617                   {
3618                     "color": "green"
3619                   },
3620                   {
3621                     "color": "red",
3622                     "value": 80
3623                   }
3624                 ]
3625               },
3626               "unit": "ms"
3627             },
3628             "overrides": []
3629           },
3630           "gridPos": {
3631             "h": 10,
3632             "w": 8,
3633             "x": 8,
3634             "y": 38
3635           },
3636           "id": 63,
3637           "options": {
3638             "legend": {
3639               "calcs": [
3640                 "mean",
3641                 "lastNotNull",
3642                 "max"
3643               ],
3644               "displayMode": "table",
3645               "placement": "bottom",
3646               "showLegend": true
3647             },
3648             "tooltip": {
3649               "mode": "single"
3650             }
3651           },
3652           "pluginVersion": "8.1.3",
3653           "targets": [
3654             {
3655               "datasource": {
3656                 "type": "prometheus",
3657                 "uid": "PBFA97CFB590B2093"
3658               },
3659               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3660               "hide": false,
3661               "legendFormat": "{{instance}} - {{quantile}}",
3662               "refId": "A"
3663             }
3664           ],
3665           "title": "Producer - ResponseQueueTimeMs",
3666           "type": "timeseries"
3667         },
3668         {
3669           "datasource": {
3670             "type": "prometheus",
3671             "uid": "PBFA97CFB590B2093"
3672           },
3673           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3674           "fieldConfig": {
3675             "defaults": {
3676               "color": {
3677                 "mode": "palette-classic"
3678               },
3679               "custom": {
3680                 "axisLabel": "",
3681                 "axisPlacement": "auto",
3682                 "barAlignment": 0,
3683                 "drawStyle": "line",
3684                 "fillOpacity": 10,
3685                 "gradientMode": "none",
3686                 "hideFrom": {
3687                   "legend": false,
3688                   "tooltip": false,
3689                   "viz": false
3690                 },
3691                 "lineInterpolation": "linear",
3692                 "lineWidth": 1,
3693                 "pointSize": 5,
3694                 "scaleDistribution": {
3695                   "type": "linear"
3696                 },
3697                 "showPoints": "never",
3698                 "spanNulls": true,
3699                 "stacking": {
3700                   "group": "A",
3701                   "mode": "none"
3702                 },
3703                 "thresholdsStyle": {
3704                   "mode": "off"
3705                 }
3706               },
3707               "links": [],
3708               "mappings": [],
3709               "min": 0,
3710               "thresholds": {
3711                 "mode": "absolute",
3712                 "steps": [
3713                   {
3714                     "color": "green"
3715                   },
3716                   {
3717                     "color": "red",
3718                     "value": 80
3719                   }
3720                 ]
3721               },
3722               "unit": "ms"
3723             },
3724             "overrides": []
3725           },
3726           "gridPos": {
3727             "h": 10,
3728             "w": 8,
3729             "x": 16,
3730             "y": 38
3731           },
3732           "id": 64,
3733           "options": {
3734             "legend": {
3735               "calcs": [
3736                 "mean",
3737                 "lastNotNull",
3738                 "max"
3739               ],
3740               "displayMode": "table",
3741               "placement": "bottom",
3742               "showLegend": true
3743             },
3744             "tooltip": {
3745               "mode": "single"
3746             }
3747           },
3748           "pluginVersion": "8.1.3",
3749           "targets": [
3750             {
3751               "datasource": {
3752                 "type": "prometheus",
3753                 "uid": "PBFA97CFB590B2093"
3754               },
3755               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3756               "hide": false,
3757               "legendFormat": "{{instance}} - {{quantile}}",
3758               "refId": "A"
3759             }
3760           ],
3761           "title": "Producer - ResponseSendTimeMs",
3762           "type": "timeseries"
3763         }
3764       ],
3765       "targets": [
3766         {
3767           "datasource": {
3768             "type": "datasource",
3769             "uid": "grafana"
3770           },
3771           "refId": "A"
3772         }
3773       ],
3774       "title": "Producer Performance",
3775       "type": "row"
3776     },
3777     {
3778       "collapsed": true,
3779       "datasource": {
3780         "type": "datasource",
3781         "uid": "grafana"
3782       },
3783       "gridPos": {
3784         "h": 1,
3785         "w": 24,
3786         "x": 0,
3787         "y": 69
3788       },
3789       "id": 68,
3790       "panels": [
3791         {
3792           "datasource": {
3793             "type": "prometheus",
3794             "uid": "PBFA97CFB590B2093"
3795           },
3796           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3797           "fieldConfig": {
3798             "defaults": {
3799               "color": {
3800                 "mode": "palette-classic"
3801               },
3802               "custom": {
3803                 "axisLabel": "",
3804                 "axisPlacement": "auto",
3805                 "barAlignment": 0,
3806                 "drawStyle": "line",
3807                 "fillOpacity": 10,
3808                 "gradientMode": "none",
3809                 "hideFrom": {
3810                   "legend": false,
3811                   "tooltip": false,
3812                   "viz": false
3813                 },
3814                 "lineInterpolation": "linear",
3815                 "lineWidth": 1,
3816                 "pointSize": 5,
3817                 "scaleDistribution": {
3818                   "type": "linear"
3819                 },
3820                 "showPoints": "never",
3821                 "spanNulls": true,
3822                 "stacking": {
3823                   "group": "A",
3824                   "mode": "none"
3825                 },
3826                 "thresholdsStyle": {
3827                   "mode": "off"
3828                 }
3829               },
3830               "links": [],
3831               "mappings": [],
3832               "min": 0,
3833               "thresholds": {
3834                 "mode": "absolute",
3835                 "steps": [
3836                   {
3837                     "color": "green"
3838                   },
3839                   {
3840                     "color": "red",
3841                     "value": 80
3842                   }
3843                 ]
3844               },
3845               "unit": "ms"
3846             },
3847             "overrides": []
3848           },
3849           "gridPos": {
3850             "h": 9,
3851             "w": 12,
3852             "x": 0,
3853             "y": 30
3854           },
3855           "id": 69,
3856           "options": {
3857             "legend": {
3858               "calcs": [
3859                 "mean",
3860                 "lastNotNull",
3861                 "max"
3862               ],
3863               "displayMode": "list",
3864               "placement": "bottom",
3865               "showLegend": true
3866             },
3867             "tooltip": {
3868               "mode": "single"
3869             }
3870           },
3871           "pluginVersion": "8.1.3",
3872           "targets": [
3873             {
3874               "datasource": {
3875                 "type": "prometheus",
3876                 "uid": "PBFA97CFB590B2093"
3877               },
3878               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3879               "hide": false,
3880               "legendFormat": "{{instance}} - {{quantile}}",
3881               "refId": "A"
3882             }
3883           ],
3884           "title": "Consumer - RequestQueueTimeMs",
3885           "type": "timeseries"
3886         },
3887         {
3888           "datasource": {
3889             "type": "prometheus",
3890             "uid": "PBFA97CFB590B2093"
3891           },
3892           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3893           "fieldConfig": {
3894             "defaults": {
3895               "color": {
3896                 "mode": "palette-classic"
3897               },
3898               "custom": {
3899                 "axisLabel": "",
3900                 "axisPlacement": "auto",
3901                 "barAlignment": 0,
3902                 "drawStyle": "line",
3903                 "fillOpacity": 10,
3904                 "gradientMode": "none",
3905                 "hideFrom": {
3906                   "legend": false,
3907                   "tooltip": false,
3908                   "viz": false
3909                 },
3910                 "lineInterpolation": "linear",
3911                 "lineWidth": 1,
3912                 "pointSize": 5,
3913                 "scaleDistribution": {
3914                   "type": "linear"
3915                 },
3916                 "showPoints": "never",
3917                 "spanNulls": true,
3918                 "stacking": {
3919                   "group": "A",
3920                   "mode": "none"
3921                 },
3922                 "thresholdsStyle": {
3923                   "mode": "off"
3924                 }
3925               },
3926               "links": [],
3927               "mappings": [],
3928               "thresholds": {
3929                 "mode": "absolute",
3930                 "steps": [
3931                   {
3932                     "color": "green"
3933                   },
3934                   {
3935                     "color": "red",
3936                     "value": 80
3937                   }
3938                 ]
3939               },
3940               "unit": "ms"
3941             },
3942             "overrides": []
3943           },
3944           "gridPos": {
3945             "h": 9,
3946             "w": 12,
3947             "x": 12,
3948             "y": 30
3949           },
3950           "id": 70,
3951           "options": {
3952             "legend": {
3953               "calcs": [
3954                 "mean",
3955                 "lastNotNull",
3956                 "max"
3957               ],
3958               "displayMode": "list",
3959               "placement": "bottom",
3960               "showLegend": true
3961             },
3962             "tooltip": {
3963               "mode": "single"
3964             }
3965           },
3966           "pluginVersion": "8.1.3",
3967           "targets": [
3968             {
3969               "datasource": {
3970                 "type": "prometheus",
3971                 "uid": "PBFA97CFB590B2093"
3972               },
3973               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3974               "hide": false,
3975               "legendFormat": "{{instance}} - {{quantile}}",
3976               "refId": "A"
3977             }
3978           ],
3979           "title": "Consumer - LocalTimeMs",
3980           "type": "timeseries"
3981         },
3982         {
3983           "datasource": {
3984             "type": "prometheus",
3985             "uid": "PBFA97CFB590B2093"
3986           },
3987           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3988           "fieldConfig": {
3989             "defaults": {
3990               "color": {
3991                 "mode": "palette-classic"
3992               },
3993               "custom": {
3994                 "axisLabel": "",
3995                 "axisPlacement": "auto",
3996                 "barAlignment": 0,
3997                 "drawStyle": "line",
3998                 "fillOpacity": 10,
3999                 "gradientMode": "none",
4000                 "hideFrom": {
4001                   "legend": false,
4002                   "tooltip": false,
4003                   "viz": false
4004                 },
4005                 "lineInterpolation": "linear",
4006                 "lineWidth": 1,
4007                 "pointSize": 5,
4008                 "scaleDistribution": {
4009                   "type": "linear"
4010                 },
4011                 "showPoints": "never",
4012                 "spanNulls": true,
4013                 "stacking": {
4014                   "group": "A",
4015                   "mode": "none"
4016                 },
4017                 "thresholdsStyle": {
4018                   "mode": "off"
4019                 }
4020               },
4021               "links": [],
4022               "mappings": [],
4023               "thresholds": {
4024                 "mode": "absolute",
4025                 "steps": [
4026                   {
4027                     "color": "green"
4028                   },
4029                   {
4030                     "color": "red",
4031                     "value": 80
4032                   }
4033                 ]
4034               },
4035               "unit": "ms"
4036             },
4037             "overrides": []
4038           },
4039           "gridPos": {
4040             "h": 9,
4041             "w": 8,
4042             "x": 0,
4043             "y": 39
4044           },
4045           "id": 71,
4046           "options": {
4047             "legend": {
4048               "calcs": [
4049                 "mean",
4050                 "lastNotNull",
4051                 "max"
4052               ],
4053               "displayMode": "table",
4054               "placement": "bottom",
4055               "showLegend": true
4056             },
4057             "tooltip": {
4058               "mode": "single"
4059             }
4060           },
4061           "pluginVersion": "8.1.3",
4062           "targets": [
4063             {
4064               "datasource": {
4065                 "type": "prometheus",
4066                 "uid": "PBFA97CFB590B2093"
4067               },
4068               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4069               "hide": false,
4070               "legendFormat": "{{instance}} - {{quantile}}",
4071               "refId": "A"
4072             }
4073           ],
4074           "title": "Consumer - RemoteTimeMs",
4075           "type": "timeseries"
4076         },
4077         {
4078           "datasource": {
4079             "type": "prometheus",
4080             "uid": "PBFA97CFB590B2093"
4081           },
4082           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4083           "fieldConfig": {
4084             "defaults": {
4085               "color": {
4086                 "mode": "palette-classic"
4087               },
4088               "custom": {
4089                 "axisLabel": "",
4090                 "axisPlacement": "auto",
4091                 "barAlignment": 0,
4092                 "drawStyle": "line",
4093                 "fillOpacity": 10,
4094                 "gradientMode": "none",
4095                 "hideFrom": {
4096                   "legend": false,
4097                   "tooltip": false,
4098                   "viz": false
4099                 },
4100                 "lineInterpolation": "linear",
4101                 "lineWidth": 1,
4102                 "pointSize": 5,
4103                 "scaleDistribution": {
4104                   "type": "linear"
4105                 },
4106                 "showPoints": "never",
4107                 "spanNulls": true,
4108                 "stacking": {
4109                   "group": "A",
4110                   "mode": "none"
4111                 },
4112                 "thresholdsStyle": {
4113                   "mode": "off"
4114                 }
4115               },
4116               "links": [],
4117               "mappings": [],
4118               "min": 0,
4119               "thresholds": {
4120                 "mode": "absolute",
4121                 "steps": [
4122                   {
4123                     "color": "green"
4124                   },
4125                   {
4126                     "color": "red",
4127                     "value": 80
4128                   }
4129                 ]
4130               },
4131               "unit": "ms"
4132             },
4133             "overrides": []
4134           },
4135           "gridPos": {
4136             "h": 9,
4137             "w": 8,
4138             "x": 8,
4139             "y": 39
4140           },
4141           "id": 72,
4142           "options": {
4143             "legend": {
4144               "calcs": [
4145                 "mean",
4146                 "lastNotNull",
4147                 "max"
4148               ],
4149               "displayMode": "table",
4150               "placement": "bottom",
4151               "showLegend": true
4152             },
4153             "tooltip": {
4154               "mode": "single"
4155             }
4156           },
4157           "pluginVersion": "8.1.3",
4158           "targets": [
4159             {
4160               "datasource": {
4161                 "type": "prometheus",
4162                 "uid": "PBFA97CFB590B2093"
4163               },
4164               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4165               "hide": false,
4166               "legendFormat": "{{instance}} - {{quantile}}",
4167               "refId": "A"
4168             }
4169           ],
4170           "title": "Consumer - ResponseQueueTimeMs",
4171           "type": "timeseries"
4172         },
4173         {
4174           "datasource": {
4175             "type": "prometheus",
4176             "uid": "PBFA97CFB590B2093"
4177           },
4178           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4179           "fieldConfig": {
4180             "defaults": {
4181               "color": {
4182                 "mode": "palette-classic"
4183               },
4184               "custom": {
4185                 "axisLabel": "",
4186                 "axisPlacement": "auto",
4187                 "barAlignment": 0,
4188                 "drawStyle": "line",
4189                 "fillOpacity": 10,
4190                 "gradientMode": "none",
4191                 "hideFrom": {
4192                   "legend": false,
4193                   "tooltip": false,
4194                   "viz": false
4195                 },
4196                 "lineInterpolation": "linear",
4197                 "lineWidth": 1,
4198                 "pointSize": 5,
4199                 "scaleDistribution": {
4200                   "type": "linear"
4201                 },
4202                 "showPoints": "never",
4203                 "spanNulls": true,
4204                 "stacking": {
4205                   "group": "A",
4206                   "mode": "none"
4207                 },
4208                 "thresholdsStyle": {
4209                   "mode": "off"
4210                 }
4211               },
4212               "links": [],
4213               "mappings": [],
4214               "min": 0,
4215               "thresholds": {
4216                 "mode": "absolute",
4217                 "steps": [
4218                   {
4219                     "color": "green"
4220                   },
4221                   {
4222                     "color": "red",
4223                     "value": 80
4224                   }
4225                 ]
4226               },
4227               "unit": "ms"
4228             },
4229             "overrides": []
4230           },
4231           "gridPos": {
4232             "h": 9,
4233             "w": 8,
4234             "x": 16,
4235             "y": 39
4236           },
4237           "id": 73,
4238           "options": {
4239             "legend": {
4240               "calcs": [
4241                 "mean",
4242                 "lastNotNull",
4243                 "max"
4244               ],
4245               "displayMode": "table",
4246               "placement": "bottom",
4247               "showLegend": true
4248             },
4249             "tooltip": {
4250               "mode": "single"
4251             }
4252           },
4253           "pluginVersion": "8.1.3",
4254           "targets": [
4255             {
4256               "datasource": {
4257                 "type": "prometheus",
4258                 "uid": "PBFA97CFB590B2093"
4259               },
4260               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4261               "hide": false,
4262               "legendFormat": "{{instance}} - {{quantile}}",
4263               "refId": "A"
4264             }
4265           ],
4266           "title": "Consumer - ResponseSendTimeMs",
4267           "type": "timeseries"
4268         }
4269       ],
4270       "targets": [
4271         {
4272           "datasource": {
4273             "type": "datasource",
4274             "uid": "grafana"
4275           },
4276           "refId": "A"
4277         }
4278       ],
4279       "title": "Consumer Performance",
4280       "type": "row"
4281     },
4282     {
4283       "collapsed": true,
4284       "datasource": {
4285         "type": "datasource",
4286         "uid": "grafana"
4287       },
4288       "gridPos": {
4289         "h": 1,
4290         "w": 24,
4291         "x": 0,
4292         "y": 70
4293       },
4294       "id": 66,
4295       "panels": [
4296         {
4297           "datasource": {
4298             "type": "prometheus",
4299             "uid": "PBFA97CFB590B2093"
4300           },
4301           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4302           "fieldConfig": {
4303             "defaults": {
4304               "color": {
4305                 "mode": "palette-classic"
4306               },
4307               "custom": {
4308                 "axisLabel": "",
4309                 "axisPlacement": "auto",
4310                 "barAlignment": 0,
4311                 "drawStyle": "line",
4312                 "fillOpacity": 10,
4313                 "gradientMode": "none",
4314                 "hideFrom": {
4315                   "legend": false,
4316                   "tooltip": false,
4317                   "viz": false
4318                 },
4319                 "lineInterpolation": "linear",
4320                 "lineWidth": 1,
4321                 "pointSize": 5,
4322                 "scaleDistribution": {
4323                   "type": "linear"
4324                 },
4325                 "showPoints": "never",
4326                 "spanNulls": true,
4327                 "stacking": {
4328                   "group": "A",
4329                   "mode": "none"
4330                 },
4331                 "thresholdsStyle": {
4332                   "mode": "off"
4333                 }
4334               },
4335               "links": [],
4336               "mappings": [],
4337               "min": 0,
4338               "thresholds": {
4339                 "mode": "absolute",
4340                 "steps": [
4341                   {
4342                     "color": "green"
4343                   },
4344                   {
4345                     "color": "red",
4346                     "value": 80
4347                   }
4348                 ]
4349               },
4350               "unit": "ms"
4351             },
4352             "overrides": []
4353           },
4354           "gridPos": {
4355             "h": 9,
4356             "w": 12,
4357             "x": 0,
4358             "y": 31
4359           },
4360           "id": 74,
4361           "options": {
4362             "legend": {
4363               "calcs": [
4364                 "mean",
4365                 "lastNotNull",
4366                 "max"
4367               ],
4368               "displayMode": "list",
4369               "placement": "bottom",
4370               "showLegend": true
4371             },
4372             "tooltip": {
4373               "mode": "single"
4374             }
4375           },
4376           "pluginVersion": "8.1.3",
4377           "targets": [
4378             {
4379               "datasource": {
4380                 "type": "prometheus",
4381                 "uid": "PBFA97CFB590B2093"
4382               },
4383               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4384               "hide": false,
4385               "legendFormat": "{{instance}} - {{quantile}}",
4386               "refId": "A"
4387             }
4388           ],
4389           "title": "FetchFollower - RequestQueueTimeMs",
4390           "type": "timeseries"
4391         },
4392         {
4393           "datasource": {
4394             "type": "prometheus",
4395             "uid": "PBFA97CFB590B2093"
4396           },
4397           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4398           "fieldConfig": {
4399             "defaults": {
4400               "color": {
4401                 "mode": "palette-classic"
4402               },
4403               "custom": {
4404                 "axisLabel": "",
4405                 "axisPlacement": "auto",
4406                 "barAlignment": 0,
4407                 "drawStyle": "line",
4408                 "fillOpacity": 10,
4409                 "gradientMode": "none",
4410                 "hideFrom": {
4411                   "legend": false,
4412                   "tooltip": false,
4413                   "viz": false
4414                 },
4415                 "lineInterpolation": "linear",
4416                 "lineWidth": 1,
4417                 "pointSize": 5,
4418                 "scaleDistribution": {
4419                   "type": "linear"
4420                 },
4421                 "showPoints": "never",
4422                 "spanNulls": true,
4423                 "stacking": {
4424                   "group": "A",
4425                   "mode": "none"
4426                 },
4427                 "thresholdsStyle": {
4428                   "mode": "off"
4429                 }
4430               },
4431               "links": [],
4432               "mappings": [],
4433               "min": 0,
4434               "thresholds": {
4435                 "mode": "absolute",
4436                 "steps": [
4437                   {
4438                     "color": "green"
4439                   },
4440                   {
4441                     "color": "red",
4442                     "value": 80
4443                   }
4444                 ]
4445               },
4446               "unit": "ms"
4447             },
4448             "overrides": []
4449           },
4450           "gridPos": {
4451             "h": 9,
4452             "w": 12,
4453             "x": 12,
4454             "y": 31
4455           },
4456           "id": 75,
4457           "options": {
4458             "legend": {
4459               "calcs": [
4460                 "mean",
4461                 "lastNotNull",
4462                 "max"
4463               ],
4464               "displayMode": "list",
4465               "placement": "bottom",
4466               "showLegend": true
4467             },
4468             "tooltip": {
4469               "mode": "single"
4470             }
4471           },
4472           "pluginVersion": "8.1.3",
4473           "targets": [
4474             {
4475               "datasource": {
4476                 "type": "prometheus",
4477                 "uid": "PBFA97CFB590B2093"
4478               },
4479               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4480               "hide": false,
4481               "legendFormat": "{{instance}} - {{quantile}}",
4482               "refId": "A"
4483             }
4484           ],
4485           "title": "FetchFollower - LocalTimeMs",
4486           "type": "timeseries"
4487         },
4488         {
4489           "datasource": {
4490             "type": "prometheus",
4491             "uid": "PBFA97CFB590B2093"
4492           },
4493           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4494           "fieldConfig": {
4495             "defaults": {
4496               "color": {
4497                 "mode": "palette-classic"
4498               },
4499               "custom": {
4500                 "axisLabel": "",
4501                 "axisPlacement": "auto",
4502                 "barAlignment": 0,
4503                 "drawStyle": "line",
4504                 "fillOpacity": 10,
4505                 "gradientMode": "none",
4506                 "hideFrom": {
4507                   "legend": false,
4508                   "tooltip": false,
4509                   "viz": false
4510                 },
4511                 "lineInterpolation": "linear",
4512                 "lineWidth": 1,
4513                 "pointSize": 5,
4514                 "scaleDistribution": {
4515                   "type": "linear"
4516                 },
4517                 "showPoints": "never",
4518                 "spanNulls": true,
4519                 "stacking": {
4520                   "group": "A",
4521                   "mode": "none"
4522                 },
4523                 "thresholdsStyle": {
4524                   "mode": "off"
4525                 }
4526               },
4527               "links": [],
4528               "mappings": [],
4529               "thresholds": {
4530                 "mode": "absolute",
4531                 "steps": [
4532                   {
4533                     "color": "green"
4534                   },
4535                   {
4536                     "color": "red",
4537                     "value": 80
4538                   }
4539                 ]
4540               },
4541               "unit": "ms"
4542             },
4543             "overrides": []
4544           },
4545           "gridPos": {
4546             "h": 9,
4547             "w": 8,
4548             "x": 0,
4549             "y": 40
4550           },
4551           "id": 76,
4552           "options": {
4553             "legend": {
4554               "calcs": [
4555                 "mean",
4556                 "lastNotNull",
4557                 "max"
4558               ],
4559               "displayMode": "table",
4560               "placement": "bottom",
4561               "showLegend": true
4562             },
4563             "tooltip": {
4564               "mode": "single"
4565             }
4566           },
4567           "pluginVersion": "8.1.3",
4568           "targets": [
4569             {
4570               "datasource": {
4571                 "type": "prometheus",
4572                 "uid": "PBFA97CFB590B2093"
4573               },
4574               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4575               "hide": false,
4576               "legendFormat": "{{instance}} - {{quantile}}",
4577               "refId": "A"
4578             }
4579           ],
4580           "title": "FetchFollower - RemoteTimeMs",
4581           "type": "timeseries"
4582         },
4583         {
4584           "datasource": {
4585             "type": "prometheus",
4586             "uid": "PBFA97CFB590B2093"
4587           },
4588           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4589           "fieldConfig": {
4590             "defaults": {
4591               "color": {
4592                 "mode": "palette-classic"
4593               },
4594               "custom": {
4595                 "axisLabel": "",
4596                 "axisPlacement": "auto",
4597                 "barAlignment": 0,
4598                 "drawStyle": "line",
4599                 "fillOpacity": 10,
4600                 "gradientMode": "none",
4601                 "hideFrom": {
4602                   "legend": false,
4603                   "tooltip": false,
4604                   "viz": false
4605                 },
4606                 "lineInterpolation": "linear",
4607                 "lineWidth": 1,
4608                 "pointSize": 5,
4609                 "scaleDistribution": {
4610                   "type": "linear"
4611                 },
4612                 "showPoints": "never",
4613                 "spanNulls": true,
4614                 "stacking": {
4615                   "group": "A",
4616                   "mode": "none"
4617                 },
4618                 "thresholdsStyle": {
4619                   "mode": "off"
4620                 }
4621               },
4622               "links": [],
4623               "mappings": [],
4624               "thresholds": {
4625                 "mode": "absolute",
4626                 "steps": [
4627                   {
4628                     "color": "green"
4629                   },
4630                   {
4631                     "color": "red",
4632                     "value": 80
4633                   }
4634                 ]
4635               },
4636               "unit": "ms"
4637             },
4638             "overrides": []
4639           },
4640           "gridPos": {
4641             "h": 9,
4642             "w": 8,
4643             "x": 8,
4644             "y": 40
4645           },
4646           "id": 77,
4647           "options": {
4648             "legend": {
4649               "calcs": [
4650                 "mean",
4651                 "lastNotNull",
4652                 "max"
4653               ],
4654               "displayMode": "table",
4655               "placement": "bottom",
4656               "showLegend": true
4657             },
4658             "tooltip": {
4659               "mode": "single"
4660             }
4661           },
4662           "pluginVersion": "8.1.3",
4663           "targets": [
4664             {
4665               "datasource": {
4666                 "type": "prometheus",
4667                 "uid": "PBFA97CFB590B2093"
4668               },
4669               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4670               "hide": false,
4671               "legendFormat": "{{instance}} - {{quantile}}",
4672               "refId": "A"
4673             }
4674           ],
4675           "title": "FetchFollower - ResponseQueueTimeMs",
4676           "type": "timeseries"
4677         },
4678         {
4679           "datasource": {
4680             "type": "prometheus",
4681             "uid": "PBFA97CFB590B2093"
4682           },
4683           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4684           "fieldConfig": {
4685             "defaults": {
4686               "color": {
4687                 "mode": "palette-classic"
4688               },
4689               "custom": {
4690                 "axisLabel": "",
4691                 "axisPlacement": "auto",
4692                 "barAlignment": 0,
4693                 "drawStyle": "line",
4694                 "fillOpacity": 10,
4695                 "gradientMode": "none",
4696                 "hideFrom": {
4697                   "legend": false,
4698                   "tooltip": false,
4699                   "viz": false
4700                 },
4701                 "lineInterpolation": "linear",
4702                 "lineWidth": 1,
4703                 "pointSize": 5,
4704                 "scaleDistribution": {
4705                   "type": "linear"
4706                 },
4707                 "showPoints": "never",
4708                 "spanNulls": true,
4709                 "stacking": {
4710                   "group": "A",
4711                   "mode": "none"
4712                 },
4713                 "thresholdsStyle": {
4714                   "mode": "off"
4715                 }
4716               },
4717               "links": [],
4718               "mappings": [],
4719               "min": 0,
4720               "thresholds": {
4721                 "mode": "absolute",
4722                 "steps": [
4723                   {
4724                     "color": "green"
4725                   },
4726                   {
4727                     "color": "red",
4728                     "value": 80
4729                   }
4730                 ]
4731               },
4732               "unit": "ms"
4733             },
4734             "overrides": []
4735           },
4736           "gridPos": {
4737             "h": 9,
4738             "w": 8,
4739             "x": 16,
4740             "y": 40
4741           },
4742           "id": 78,
4743           "options": {
4744             "legend": {
4745               "calcs": [
4746                 "mean",
4747                 "lastNotNull",
4748                 "max"
4749               ],
4750               "displayMode": "table",
4751               "placement": "bottom",
4752               "showLegend": true
4753             },
4754             "tooltip": {
4755               "mode": "single"
4756             }
4757           },
4758           "pluginVersion": "8.1.3",
4759           "targets": [
4760             {
4761               "datasource": {
4762                 "type": "prometheus",
4763                 "uid": "PBFA97CFB590B2093"
4764               },
4765               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4766               "hide": false,
4767               "legendFormat": "{{instance}} - {{quantile}}",
4768               "refId": "A"
4769             }
4770           ],
4771           "title": "FetchFollower - ResponseSendTimeMs",
4772           "type": "timeseries"
4773         }
4774       ],
4775       "targets": [
4776         {
4777           "datasource": {
4778             "type": "datasource",
4779             "uid": "grafana"
4780           },
4781           "refId": "A"
4782         }
4783       ],
4784       "title": "Fetch Follower Performance",
4785       "type": "row"
4786     },
4787     {
4788       "collapsed": true,
4789       "datasource": {
4790         "type": "datasource",
4791         "uid": "grafana"
4792       },
4793       "gridPos": {
4794         "h": 1,
4795         "w": 24,
4796         "x": 0,
4797         "y": 71
4798       },
4799       "id": 102,
4800       "panels": [
4801         {
4802           "datasource": {
4803             "type": "prometheus",
4804             "uid": "PBFA97CFB590B2093"
4805           },
4806           "fieldConfig": {
4807             "defaults": {
4808               "color": {
4809                 "mode": "palette-classic"
4810               },
4811               "custom": {
4812                 "axisLabel": "",
4813                 "axisPlacement": "auto",
4814                 "barAlignment": 0,
4815                 "drawStyle": "line",
4816                 "fillOpacity": 10,
4817                 "gradientMode": "none",
4818                 "hideFrom": {
4819                   "legend": false,
4820                   "tooltip": false,
4821                   "viz": false
4822                 },
4823                 "lineInterpolation": "linear",
4824                 "lineWidth": 1,
4825                 "pointSize": 5,
4826                 "scaleDistribution": {
4827                   "type": "linear"
4828                 },
4829                 "showPoints": "never",
4830                 "spanNulls": true,
4831                 "stacking": {
4832                   "group": "A",
4833                   "mode": "none"
4834                 },
4835                 "thresholdsStyle": {
4836                   "mode": "off"
4837                 }
4838               },
4839               "links": [],
4840               "mappings": [],
4841               "thresholds": {
4842                 "mode": "absolute",
4843                 "steps": [
4844                   {
4845                     "color": "green"
4846                   },
4847                   {
4848                     "color": "red",
4849                     "value": 80
4850                   }
4851                 ]
4852               },
4853               "unit": "short"
4854             },
4855             "overrides": []
4856           },
4857           "gridPos": {
4858             "h": 8,
4859             "w": 12,
4860             "x": 0,
4861             "y": 32
4862           },
4863           "id": 98,
4864           "options": {
4865             "legend": {
4866               "calcs": [
4867                 "mean",
4868                 "lastNotNull",
4869                 "max"
4870               ],
4871               "displayMode": "table",
4872               "placement": "bottom",
4873               "showLegend": true
4874             },
4875             "tooltip": {
4876               "mode": "single"
4877             }
4878           },
4879           "pluginVersion": "8.1.3",
4880           "targets": [
4881             {
4882               "datasource": {
4883                 "type": "prometheus",
4884                 "uid": "PBFA97CFB590B2093"
4885               },
4886               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4887               "interval": "",
4888               "legendFormat": "{{listener}}",
4889               "refId": "A"
4890             }
4891           ],
4892           "title": "Connections count per listener",
4893           "type": "timeseries"
4894         },
4895         {
4896           "datasource": {
4897             "type": "prometheus",
4898             "uid": "PBFA97CFB590B2093"
4899           },
4900           "fieldConfig": {
4901             "defaults": {
4902               "color": {
4903                 "mode": "palette-classic"
4904               },
4905               "custom": {
4906                 "axisLabel": "",
4907                 "axisPlacement": "auto",
4908                 "barAlignment": 0,
4909                 "drawStyle": "line",
4910                 "fillOpacity": 10,
4911                 "gradientMode": "none",
4912                 "hideFrom": {
4913                   "legend": false,
4914                   "tooltip": false,
4915                   "viz": false
4916                 },
4917                 "lineInterpolation": "linear",
4918                 "lineWidth": 1,
4919                 "pointSize": 5,
4920                 "scaleDistribution": {
4921                   "type": "linear"
4922                 },
4923                 "showPoints": "never",
4924                 "spanNulls": true,
4925                 "stacking": {
4926                   "group": "A",
4927                   "mode": "none"
4928                 },
4929                 "thresholdsStyle": {
4930                   "mode": "off"
4931                 }
4932               },
4933               "links": [],
4934               "mappings": [],
4935               "thresholds": {
4936                 "mode": "absolute",
4937                 "steps": [
4938                   {
4939                     "color": "green"
4940                   },
4941                   {
4942                     "color": "red",
4943                     "value": 80
4944                   }
4945                 ]
4946               },
4947               "unit": "short"
4948             },
4949             "overrides": []
4950           },
4951           "gridPos": {
4952             "h": 8,
4953             "w": 12,
4954             "x": 12,
4955             "y": 32
4956           },
4957           "id": 100,
4958           "options": {
4959             "legend": {
4960               "calcs": [
4961                 "mean",
4962                 "lastNotNull",
4963                 "max"
4964               ],
4965               "displayMode": "table",
4966               "placement": "bottom",
4967               "showLegend": true
4968             },
4969             "tooltip": {
4970               "mode": "single"
4971             }
4972           },
4973           "pluginVersion": "8.1.3",
4974           "targets": [
4975             {
4976               "datasource": {
4977                 "type": "prometheus",
4978                 "uid": "PBFA97CFB590B2093"
4979               },
4980               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
4981               "interval": "",
4982               "legendFormat": "{{instance}}",
4983               "refId": "A"
4984             }
4985           ],
4986           "title": "Connections count per broker",
4987           "type": "timeseries"
4988         },
4989         {
4990           "datasource": {
4991             "type": "prometheus",
4992             "uid": "PBFA97CFB590B2093"
4993           },
4994           "fieldConfig": {
4995             "defaults": {
4996               "color": {
4997                 "mode": "palette-classic"
4998               },
4999               "custom": {
5000                 "axisLabel": "",
5001                 "axisPlacement": "auto",
5002                 "barAlignment": 0,
5003                 "drawStyle": "line",
5004                 "fillOpacity": 10,
5005                 "gradientMode": "none",
5006                 "hideFrom": {
5007                   "legend": false,
5008                   "tooltip": false,
5009                   "viz": false
5010                 },
5011                 "lineInterpolation": "linear",
5012                 "lineWidth": 1,
5013                 "pointSize": 5,
5014                 "scaleDistribution": {
5015                   "type": "linear"
5016                 },
5017                 "showPoints": "never",
5018                 "spanNulls": true,
5019                 "stacking": {
5020                   "group": "A",
5021                   "mode": "none"
5022                 },
5023                 "thresholdsStyle": {
5024                   "mode": "off"
5025                 }
5026               },
5027               "links": [],
5028               "mappings": [],
5029               "thresholds": {
5030                 "mode": "absolute",
5031                 "steps": [
5032                   {
5033                     "color": "green"
5034                   },
5035                   {
5036                     "color": "red",
5037                     "value": 80
5038                   }
5039                 ]
5040               },
5041               "unit": "short"
5042             },
5043             "overrides": []
5044           },
5045           "gridPos": {
5046             "h": 8,
5047             "w": 12,
5048             "x": 0,
5049             "y": 40
5050           },
5051           "id": 104,
5052           "options": {
5053             "legend": {
5054               "calcs": [
5055                 "mean",
5056                 "lastNotNull",
5057                 "max"
5058               ],
5059               "displayMode": "table",
5060               "placement": "bottom",
5061               "showLegend": true
5062             },
5063             "tooltip": {
5064               "mode": "single"
5065             }
5066           },
5067           "pluginVersion": "8.1.3",
5068           "targets": [
5069             {
5070               "datasource": {
5071                 "type": "prometheus",
5072                 "uid": "PBFA97CFB590B2093"
5073               },
5074               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5075               "interval": "",
5076               "legendFormat": "{{listener}}",
5077               "refId": "A"
5078             }
5079           ],
5080           "title": "Connections creation rate per listener",
5081           "type": "timeseries"
5082         },
5083         {
5084           "datasource": {
5085             "type": "prometheus",
5086             "uid": "PBFA97CFB590B2093"
5087           },
5088           "fieldConfig": {
5089             "defaults": {
5090               "color": {
5091                 "mode": "palette-classic"
5092               },
5093               "custom": {
5094                 "axisLabel": "",
5095                 "axisPlacement": "auto",
5096                 "barAlignment": 0,
5097                 "drawStyle": "line",
5098                 "fillOpacity": 10,
5099                 "gradientMode": "none",
5100                 "hideFrom": {
5101                   "legend": false,
5102                   "tooltip": false,
5103                   "viz": false
5104                 },
5105                 "lineInterpolation": "linear",
5106                 "lineWidth": 1,
5107                 "pointSize": 5,
5108                 "scaleDistribution": {
5109                   "type": "linear"
5110                 },
5111                 "showPoints": "never",
5112                 "spanNulls": true,
5113                 "stacking": {
5114                   "group": "A",
5115                   "mode": "none"
5116                 },
5117                 "thresholdsStyle": {
5118                   "mode": "off"
5119                 }
5120               },
5121               "links": [],
5122               "mappings": [],
5123               "thresholds": {
5124                 "mode": "absolute",
5125                 "steps": [
5126                   {
5127                     "color": "green"
5128                   },
5129                   {
5130                     "color": "red",
5131                     "value": 80
5132                   }
5133                 ]
5134               },
5135               "unit": "short"
5136             },
5137             "overrides": []
5138           },
5139           "gridPos": {
5140             "h": 8,
5141             "w": 12,
5142             "x": 12,
5143             "y": 40
5144           },
5145           "id": 106,
5146           "options": {
5147             "legend": {
5148               "calcs": [
5149                 "mean",
5150                 "lastNotNull",
5151                 "max"
5152               ],
5153               "displayMode": "table",
5154               "placement": "bottom",
5155               "showLegend": true
5156             },
5157             "tooltip": {
5158               "mode": "single"
5159             }
5160           },
5161           "pluginVersion": "8.1.3",
5162           "targets": [
5163             {
5164               "datasource": {
5165                 "type": "prometheus",
5166                 "uid": "PBFA97CFB590B2093"
5167               },
5168               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5169               "interval": "",
5170               "legendFormat": "{{instance}}",
5171               "refId": "A"
5172             }
5173           ],
5174           "title": "Connections creation rate per instance",
5175           "type": "timeseries"
5176         },
5177         {
5178           "datasource": {
5179             "type": "prometheus",
5180             "uid": "PBFA97CFB590B2093"
5181           },
5182           "fieldConfig": {
5183             "defaults": {
5184               "color": {
5185                 "mode": "palette-classic"
5186               },
5187               "custom": {
5188                 "axisLabel": "",
5189                 "axisPlacement": "auto",
5190                 "barAlignment": 0,
5191                 "drawStyle": "line",
5192                 "fillOpacity": 10,
5193                 "gradientMode": "none",
5194                 "hideFrom": {
5195                   "legend": false,
5196                   "tooltip": false,
5197                   "viz": false
5198                 },
5199                 "lineInterpolation": "linear",
5200                 "lineWidth": 1,
5201                 "pointSize": 5,
5202                 "scaleDistribution": {
5203                   "type": "linear"
5204                 },
5205                 "showPoints": "never",
5206                 "spanNulls": true,
5207                 "stacking": {
5208                   "group": "A",
5209                   "mode": "none"
5210                 },
5211                 "thresholdsStyle": {
5212                   "mode": "off"
5213                 }
5214               },
5215               "links": [],
5216               "mappings": [],
5217               "thresholds": {
5218                 "mode": "absolute",
5219                 "steps": [
5220                   {
5221                     "color": "green"
5222                   },
5223                   {
5224                     "color": "red",
5225                     "value": 80
5226                   }
5227                 ]
5228               },
5229               "unit": "short"
5230             },
5231             "overrides": []
5232           },
5233           "gridPos": {
5234             "h": 8,
5235             "w": 12,
5236             "x": 0,
5237             "y": 48
5238           },
5239           "id": 108,
5240           "options": {
5241             "legend": {
5242               "calcs": [
5243                 "mean",
5244                 "lastNotNull",
5245                 "max"
5246               ],
5247               "displayMode": "table",
5248               "placement": "bottom",
5249               "showLegend": true
5250             },
5251             "tooltip": {
5252               "mode": "single"
5253             }
5254           },
5255           "pluginVersion": "8.1.3",
5256           "targets": [
5257             {
5258               "datasource": {
5259                 "type": "prometheus",
5260                 "uid": "PBFA97CFB590B2093"
5261               },
5262               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5263               "interval": "",
5264               "legendFormat": "{{listener}}",
5265               "refId": "A"
5266             }
5267           ],
5268           "title": "Connections close rate per listener",
5269           "type": "timeseries"
5270         },
5271         {
5272           "datasource": {
5273             "type": "prometheus",
5274             "uid": "PBFA97CFB590B2093"
5275           },
5276           "fieldConfig": {
5277             "defaults": {
5278               "color": {
5279                 "mode": "palette-classic"
5280               },
5281               "custom": {
5282                 "axisLabel": "",
5283                 "axisPlacement": "auto",
5284                 "barAlignment": 0,
5285                 "drawStyle": "line",
5286                 "fillOpacity": 10,
5287                 "gradientMode": "none",
5288                 "hideFrom": {
5289                   "legend": false,
5290                   "tooltip": false,
5291                   "viz": false
5292                 },
5293                 "lineInterpolation": "linear",
5294                 "lineWidth": 1,
5295                 "pointSize": 5,
5296                 "scaleDistribution": {
5297                   "type": "linear"
5298                 },
5299                 "showPoints": "never",
5300                 "spanNulls": true,
5301                 "stacking": {
5302                   "group": "A",
5303                   "mode": "none"
5304                 },
5305                 "thresholdsStyle": {
5306                   "mode": "off"
5307                 }
5308               },
5309               "links": [],
5310               "mappings": [],
5311               "thresholds": {
5312                 "mode": "absolute",
5313                 "steps": [
5314                   {
5315                     "color": "green"
5316                   },
5317                   {
5318                     "color": "red",
5319                     "value": 80
5320                   }
5321                 ]
5322               },
5323               "unit": "short"
5324             },
5325             "overrides": []
5326           },
5327           "gridPos": {
5328             "h": 8,
5329             "w": 12,
5330             "x": 12,
5331             "y": 48
5332           },
5333           "id": 110,
5334           "options": {
5335             "legend": {
5336               "calcs": [
5337                 "mean",
5338                 "lastNotNull",
5339                 "max"
5340               ],
5341               "displayMode": "table",
5342               "placement": "bottom",
5343               "showLegend": true
5344             },
5345             "tooltip": {
5346               "mode": "single"
5347             }
5348           },
5349           "pluginVersion": "8.1.3",
5350           "targets": [
5351             {
5352               "datasource": {
5353                 "type": "prometheus",
5354                 "uid": "PBFA97CFB590B2093"
5355               },
5356               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5357               "interval": "",
5358               "legendFormat": "{{instance}}",
5359               "refId": "A"
5360             }
5361           ],
5362           "title": "Connections close rate per instance",
5363           "type": "timeseries"
5364         },
5365         {
5366           "datasource": {
5367             "type": "prometheus",
5368             "uid": "PBFA97CFB590B2093"
5369           },
5370           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5371           "fieldConfig": {
5372             "defaults": {
5373               "color": {
5374                 "mode": "palette-classic"
5375               },
5376               "custom": {
5377                 "axisLabel": "",
5378                 "axisPlacement": "auto",
5379                 "barAlignment": 0,
5380                 "drawStyle": "line",
5381                 "fillOpacity": 10,
5382                 "gradientMode": "none",
5383                 "hideFrom": {
5384                   "legend": false,
5385                   "tooltip": false,
5386                   "viz": false
5387                 },
5388                 "lineInterpolation": "linear",
5389                 "lineWidth": 1,
5390                 "pointSize": 5,
5391                 "scaleDistribution": {
5392                   "type": "linear"
5393                 },
5394                 "showPoints": "never",
5395                 "spanNulls": true,
5396                 "stacking": {
5397                   "group": "A",
5398                   "mode": "none"
5399                 },
5400                 "thresholdsStyle": {
5401                   "mode": "off"
5402                 }
5403               },
5404               "links": [],
5405               "mappings": [],
5406               "min": 0,
5407               "thresholds": {
5408                 "mode": "absolute",
5409                 "steps": [
5410                   {
5411                     "color": "green"
5412                   },
5413                   {
5414                     "color": "red",
5415                     "value": 80
5416                   }
5417                 ]
5418               },
5419               "unit": "percent"
5420             },
5421             "overrides": []
5422           },
5423           "gridPos": {
5424             "h": 8,
5425             "w": 12,
5426             "x": 0,
5427             "y": 56
5428           },
5429           "id": 112,
5430           "options": {
5431             "legend": {
5432               "calcs": [
5433                 "mean",
5434                 "lastNotNull",
5435                 "max"
5436               ],
5437               "displayMode": "table",
5438               "placement": "bottom",
5439               "showLegend": true
5440             },
5441             "tooltip": {
5442               "mode": "single"
5443             }
5444           },
5445           "pluginVersion": "8.1.3",
5446           "targets": [
5447             {
5448               "datasource": {
5449                 "type": "prometheus",
5450                 "uid": "PBFA97CFB590B2093"
5451               },
5452               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5453               "interval": "",
5454               "legendFormat": "{{instance}} - {{listener}}",
5455               "refId": "A"
5456             }
5457           ],
5458           "title": "Acceptor Blocked Percentage",
5459           "type": "timeseries"
5460         },
5461         {
5462           "datasource": {
5463             "type": "prometheus",
5464             "uid": "PBFA97CFB590B2093"
5465           },
5466           "fieldConfig": {
5467             "defaults": {
5468               "color": {
5469                 "mode": "palette-classic"
5470               },
5471               "custom": {
5472                 "axisLabel": "",
5473                 "axisPlacement": "auto",
5474                 "barAlignment": 0,
5475                 "drawStyle": "line",
5476                 "fillOpacity": 10,
5477                 "gradientMode": "none",
5478                 "hideFrom": {
5479                   "legend": false,
5480                   "tooltip": false,
5481                   "viz": false
5482                 },
5483                 "lineInterpolation": "linear",
5484                 "lineWidth": 1,
5485                 "pointSize": 5,
5486                 "scaleDistribution": {
5487                   "type": "linear"
5488                 },
5489                 "showPoints": "never",
5490                 "spanNulls": true,
5491                 "stacking": {
5492                   "group": "A",
5493                   "mode": "none"
5494                 },
5495                 "thresholdsStyle": {
5496                   "mode": "off"
5497                 }
5498               },
5499               "links": [],
5500               "mappings": [],
5501               "thresholds": {
5502                 "mode": "absolute",
5503                 "steps": [
5504                   {
5505                     "color": "green"
5506                   },
5507                   {
5508                     "color": "red",
5509                     "value": 80
5510                   }
5511                 ]
5512               },
5513               "unit": "short"
5514             },
5515             "overrides": []
5516           },
5517           "gridPos": {
5518             "h": 8,
5519             "w": 12,
5520             "x": 12,
5521             "y": 56
5522           },
5523           "id": 114,
5524           "options": {
5525             "legend": {
5526               "calcs": [
5527                 "mean",
5528                 "lastNotNull",
5529                 "max"
5530               ],
5531               "displayMode": "table",
5532               "placement": "bottom",
5533               "showLegend": true
5534             },
5535             "tooltip": {
5536               "mode": "single"
5537             }
5538           },
5539           "pluginVersion": "8.1.3",
5540           "targets": [
5541             {
5542               "datasource": {
5543                 "type": "prometheus",
5544                 "uid": "PBFA97CFB590B2093"
5545               },
5546               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5547               "interval": "",
5548               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5549               "refId": "A"
5550             }
5551           ],
5552           "title": "Connections per client version",
5553           "type": "timeseries"
5554         }
5555       ],
5556       "targets": [
5557         {
5558           "datasource": {
5559             "type": "datasource",
5560             "uid": "grafana"
5561           },
5562           "refId": "A"
5563         }
5564       ],
5565       "title": "Connections",
5566       "type": "row"
5567     },
5568     {
5569       "collapsed": true,
5570       "datasource": {
5571         "type": "datasource",
5572         "uid": "grafana"
5573       },
5574       "gridPos": {
5575         "h": 1,
5576         "w": 24,
5577         "x": 0,
5578         "y": 72
5579       },
5580       "id": 120,
5581       "panels": [
5582         {
5583           "datasource": {
5584             "type": "prometheus",
5585             "uid": "PBFA97CFB590B2093"
5586           },
5587           "description": "Number of consumer groups per group coordinator",
5588           "fieldConfig": {
5589             "defaults": {
5590               "color": {
5591                 "mode": "palette-classic"
5592               },
5593               "custom": {
5594                 "axisLabel": "",
5595                 "axisPlacement": "auto",
5596                 "barAlignment": 0,
5597                 "drawStyle": "line",
5598                 "fillOpacity": 10,
5599                 "gradientMode": "none",
5600                 "hideFrom": {
5601                   "legend": false,
5602                   "tooltip": false,
5603                   "viz": false
5604                 },
5605                 "lineInterpolation": "linear",
5606                 "lineWidth": 1,
5607                 "pointSize": 5,
5608                 "scaleDistribution": {
5609                   "type": "linear"
5610                 },
5611                 "showPoints": "never",
5612                 "spanNulls": true,
5613                 "stacking": {
5614                   "group": "A",
5615                   "mode": "none"
5616                 },
5617                 "thresholdsStyle": {
5618                   "mode": "off"
5619                 }
5620               },
5621               "links": [],
5622               "mappings": [],
5623               "thresholds": {
5624                 "mode": "absolute",
5625                 "steps": [
5626                   {
5627                     "color": "green"
5628                   },
5629                   {
5630                     "color": "red",
5631                     "value": 80
5632                   }
5633                 ]
5634               },
5635               "unit": "short"
5636             },
5637             "overrides": []
5638           },
5639           "gridPos": {
5640             "h": 8,
5641             "w": 12,
5642             "x": 0,
5643             "y": 33
5644           },
5645           "id": 116,
5646           "options": {
5647             "legend": {
5648               "calcs": [
5649                 "mean",
5650                 "lastNotNull",
5651                 "max"
5652               ],
5653               "displayMode": "table",
5654               "placement": "bottom",
5655               "showLegend": true
5656             },
5657             "tooltip": {
5658               "mode": "single"
5659             }
5660           },
5661           "pluginVersion": "8.1.3",
5662           "targets": [
5663             {
5664               "datasource": {
5665                 "type": "prometheus",
5666                 "uid": "PBFA97CFB590B2093"
5667               },
5668               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5669               "instant": false,
5670               "interval": "",
5671               "legendFormat": "{{instance}}",
5672               "refId": "A"
5673             }
5674           ],
5675           "title": "Consumer groups number per coordinator",
5676           "type": "timeseries"
5677         },
5678         {
5679           "datasource": {
5680             "type": "prometheus",
5681             "uid": "PBFA97CFB590B2093"
5682           },
5683           "description": "Number of consumer group per state",
5684           "fieldConfig": {
5685             "defaults": {
5686               "color": {
5687                 "mode": "palette-classic"
5688               },
5689               "custom": {
5690                 "axisLabel": "",
5691                 "axisPlacement": "auto",
5692                 "barAlignment": 0,
5693                 "drawStyle": "line",
5694                 "fillOpacity": 10,
5695                 "gradientMode": "none",
5696                 "hideFrom": {
5697                   "legend": false,
5698                   "tooltip": false,
5699                   "viz": false
5700                 },
5701                 "lineInterpolation": "linear",
5702                 "lineWidth": 1,
5703                 "pointSize": 5,
5704                 "scaleDistribution": {
5705                   "type": "linear"
5706                 },
5707                 "showPoints": "never",
5708                 "spanNulls": true,
5709                 "stacking": {
5710                   "group": "A",
5711                   "mode": "none"
5712                 },
5713                 "thresholdsStyle": {
5714                   "mode": "off"
5715                 }
5716               },
5717               "links": [],
5718               "mappings": [],
5719               "thresholds": {
5720                 "mode": "absolute",
5721                 "steps": [
5722                   {
5723                     "color": "green"
5724                   },
5725                   {
5726                     "color": "red",
5727                     "value": 80
5728                   }
5729                 ]
5730               },
5731               "unit": "short"
5732             },
5733             "overrides": []
5734           },
5735           "gridPos": {
5736             "h": 8,
5737             "w": 12,
5738             "x": 12,
5739             "y": 33
5740           },
5741           "id": 118,
5742           "options": {
5743             "legend": {
5744               "calcs": [
5745                 "mean",
5746                 "lastNotNull",
5747                 "max"
5748               ],
5749               "displayMode": "table",
5750               "placement": "bottom",
5751               "showLegend": true
5752             },
5753             "tooltip": {
5754               "mode": "single"
5755             }
5756           },
5757           "pluginVersion": "8.1.3",
5758           "targets": [
5759             {
5760               "datasource": {
5761                 "type": "prometheus",
5762                 "uid": "PBFA97CFB590B2093"
5763               },
5764               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5765               "instant": false,
5766               "interval": "",
5767               "legendFormat": "stable",
5768               "refId": "A"
5769             },
5770             {
5771               "datasource": {
5772                 "type": "prometheus",
5773                 "uid": "PBFA97CFB590B2093"
5774               },
5775               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5776               "interval": "",
5777               "legendFormat": "preparing-rebalance",
5778               "refId": "B"
5779             },
5780             {
5781               "datasource": {
5782                 "type": "prometheus",
5783                 "uid": "PBFA97CFB590B2093"
5784               },
5785               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5786               "interval": "",
5787               "legendFormat": "dead",
5788               "refId": "C"
5789             },
5790             {
5791               "datasource": {
5792                 "type": "prometheus",
5793                 "uid": "PBFA97CFB590B2093"
5794               },
5795               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5796               "interval": "",
5797               "legendFormat": "completing-rebalance",
5798               "refId": "D"
5799             },
5800             {
5801               "datasource": {
5802                 "type": "prometheus",
5803                 "uid": "PBFA97CFB590B2093"
5804               },
5805               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5806               "interval": "",
5807               "legendFormat": "empty",
5808               "refId": "E"
5809             }
5810           ],
5811           "title": "Nb consumer groups per state",
5812           "type": "timeseries"
5813         }
5814       ],
5815       "targets": [
5816         {
5817           "datasource": {
5818             "type": "datasource",
5819             "uid": "grafana"
5820           },
5821           "refId": "A"
5822         }
5823       ],
5824       "title": "Group Coordinator",
5825       "type": "row"
5826     },
5827     {
5828       "collapsed": true,
5829       "datasource": {
5830         "type": "datasource",
5831         "uid": "grafana"
5832       },
5833       "gridPos": {
5834         "h": 1,
5835         "w": 24,
5836         "x": 0,
5837         "y": 73
5838       },
5839       "id": 46,
5840       "panels": [
5841         {
5842           "datasource": {
5843             "type": "prometheus",
5844             "uid": "PBFA97CFB590B2093"
5845           },
5846           "description": "The number of messages produced converted to match the log.message.format.version.",
5847           "fieldConfig": {
5848             "defaults": {
5849               "color": {
5850                 "mode": "palette-classic"
5851               },
5852               "custom": {
5853                 "axisLabel": "",
5854                 "axisPlacement": "auto",
5855                 "barAlignment": 0,
5856                 "drawStyle": "line",
5857                 "fillOpacity": 10,
5858                 "gradientMode": "none",
5859                 "hideFrom": {
5860                   "legend": false,
5861                   "tooltip": false,
5862                   "viz": false
5863                 },
5864                 "lineInterpolation": "linear",
5865                 "lineWidth": 1,
5866                 "pointSize": 5,
5867                 "scaleDistribution": {
5868                   "type": "linear"
5869                 },
5870                 "showPoints": "never",
5871                 "spanNulls": true,
5872                 "stacking": {
5873                   "group": "A",
5874                   "mode": "none"
5875                 },
5876                 "thresholdsStyle": {
5877                   "mode": "off"
5878                 }
5879               },
5880               "links": [],
5881               "mappings": [],
5882               "min": 0,
5883               "thresholds": {
5884                 "mode": "absolute",
5885                 "steps": [
5886                   {
5887                     "color": "green"
5888                   },
5889                   {
5890                     "color": "red",
5891                     "value": 80
5892                   }
5893                 ]
5894               },
5895               "unit": "ops"
5896             },
5897             "overrides": []
5898           },
5899           "gridPos": {
5900             "h": 8,
5901             "w": 8,
5902             "x": 0,
5903             "y": 34
5904           },
5905           "id": 48,
5906           "options": {
5907             "legend": {
5908               "calcs": [
5909                 "mean",
5910                 "lastNotNull",
5911                 "max"
5912               ],
5913               "displayMode": "list",
5914               "placement": "bottom",
5915               "showLegend": true
5916             },
5917             "tooltip": {
5918               "mode": "single"
5919             }
5920           },
5921           "pluginVersion": "8.1.3",
5922           "targets": [
5923             {
5924               "datasource": {
5925                 "type": "prometheus",
5926                 "uid": "PBFA97CFB590B2093"
5927               },
5928               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5929               "hide": false,
5930               "interval": "",
5931               "legendFormat": "{{topic}}",
5932               "refId": "A"
5933             }
5934           ],
5935           "title": "Number of produced message conversion",
5936           "type": "timeseries"
5937         },
5938         {
5939           "datasource": {
5940             "type": "prometheus",
5941             "uid": "PBFA97CFB590B2093"
5942           },
5943           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5944           "fieldConfig": {
5945             "defaults": {
5946               "color": {
5947                 "mode": "palette-classic"
5948               },
5949               "custom": {
5950                 "axisLabel": "",
5951                 "axisPlacement": "auto",
5952                 "barAlignment": 0,
5953                 "drawStyle": "line",
5954                 "fillOpacity": 10,
5955                 "gradientMode": "none",
5956                 "hideFrom": {
5957                   "legend": false,
5958                   "tooltip": false,
5959                   "viz": false
5960                 },
5961                 "lineInterpolation": "linear",
5962                 "lineWidth": 1,
5963                 "pointSize": 5,
5964                 "scaleDistribution": {
5965                   "type": "linear"
5966                 },
5967                 "showPoints": "never",
5968                 "spanNulls": true,
5969                 "stacking": {
5970                   "group": "A",
5971                   "mode": "none"
5972                 },
5973                 "thresholdsStyle": {
5974                   "mode": "off"
5975                 }
5976               },
5977               "links": [],
5978               "mappings": [],
5979               "min": 0,
5980               "thresholds": {
5981                 "mode": "absolute",
5982                 "steps": [
5983                   {
5984                     "color": "green"
5985                   },
5986                   {
5987                     "color": "red",
5988                     "value": 80
5989                   }
5990                 ]
5991               },
5992               "unit": "ops"
5993             },
5994             "overrides": []
5995           },
5996           "gridPos": {
5997             "h": 8,
5998             "w": 8,
5999             "x": 8,
6000             "y": 34
6001           },
6002           "id": 51,
6003           "options": {
6004             "legend": {
6005               "calcs": [
6006                 "mean",
6007                 "lastNotNull",
6008                 "max"
6009               ],
6010               "displayMode": "list",
6011               "placement": "bottom",
6012               "showLegend": true
6013             },
6014             "tooltip": {
6015               "mode": "single"
6016             }
6017           },
6018           "pluginVersion": "8.1.3",
6019           "targets": [
6020             {
6021               "datasource": {
6022                 "type": "prometheus",
6023                 "uid": "PBFA97CFB590B2093"
6024               },
6025               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6026               "hide": false,
6027               "interval": "",
6028               "legendFormat": "{{topic}}",
6029               "refId": "A"
6030             }
6031           ],
6032           "title": "Number of consumed message conversion",
6033           "type": "timeseries"
6034         },
6035         {
6036           "datasource": {
6037             "type": "datasource",
6038             "uid": "grafana"
6039           },
6040           "description": "Number of connection per client version",
6041           "fieldConfig": {
6042             "defaults": {
6043               "color": {
6044                 "mode": "palette-classic"
6045               },
6046               "custom": {
6047                 "hideFrom": {
6048                   "legend": false,
6049                   "tooltip": false,
6050                   "viz": false
6051                 }
6052               },
6053               "decimals": 0,
6054               "mappings": [],
6055               "unit": "short"
6056             },
6057             "overrides": []
6058           },
6059           "gridPos": {
6060             "h": 8,
6061             "w": 8,
6062             "x": 16,
6063             "y": 34
6064           },
6065           "id": 96,
6066           "links": [],
6067           "options": {
6068             "legend": {
6069               "calcs": [],
6070               "displayMode": "table",
6071               "placement": "bottom",
6072               "showLegend": true,
6073               "values": [
6074                 "value"
6075               ]
6076             },
6077             "pieType": "pie",
6078             "reduceOptions": {
6079               "calcs": [
6080                 "lastNotNull"
6081               ],
6082               "fields": "",
6083               "values": false
6084             },
6085             "tooltip": {
6086               "mode": "single"
6087             }
6088           },
6089           "targets": [
6090             {
6091               "datasource": {
6092                 "type": "datasource",
6093                 "uid": "grafana"
6094               },
6095               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6096               "interval": "",
6097               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6098               "refId": "A"
6099             }
6100           ],
6101           "title": "Client version repartition",
6102           "type": "piechart"
6103         }
6104       ],
6105       "targets": [
6106         {
6107           "datasource": {
6108             "type": "datasource",
6109             "uid": "grafana"
6110           },
6111           "refId": "A"
6112         }
6113       ],
6114       "title": "Message Conversion",
6115       "type": "row"
6116     }
6117   ],
6118   "refresh": "",
6119   "schemaVersion": 38,
6120   "style": "dark",
6121   "tags": [],
6122   "templating": {
6123     "list": [
6124       {
6125         "current": {
6126           "selected": false,
6127           "text": "dev",
6128           "value": "dev"
6129         },
6130         "datasource": {
6131           "type": "prometheus",
6132           "uid": "PBFA97CFB590B2093"
6133         },
6134         "definition": "label_values(env)",
6135         "hide": 0,
6136         "includeAll": false,
6137         "label": "Environment",
6138         "multi": false,
6139         "name": "env",
6140         "options": [],
6141         "query": {
6142           "query": "label_values(env)",
6143           "refId": "Prometheus-env-Variable-Query"
6144         },
6145         "refresh": 1,
6146         "regex": "",
6147         "skipUrlSync": false,
6148         "sort": 0,
6149         "tagValuesQuery": "",
6150         "tagsQuery": "",
6151         "type": "query",
6152         "useTags": false
6153       },
6154       {
6155         "current": {
6156           "selected": true,
6157           "text": [
6158             "All"
6159           ],
6160           "value": [
6161             "$__all"
6162           ]
6163         },
6164         "datasource": {
6165           "type": "prometheus",
6166           "uid": "PBFA97CFB590B2093"
6167         },
6168         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6169         "hide": 0,
6170         "includeAll": true,
6171         "label": "Instance",
6172         "multi": true,
6173         "name": "instance",
6174         "options": [],
6175         "query": {
6176           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6177           "refId": "Prometheus-instance-Variable-Query"
6178         },
6179         "refresh": 1,
6180         "regex": "",
6181         "skipUrlSync": false,
6182         "sort": 0,
6183         "tagValuesQuery": "",
6184         "tagsQuery": "",
6185         "type": "query",
6186         "useTags": false
6187       },
6188       {
6189         "allValue": "",
6190         "current": {
6191           "selected": true,
6192           "text": [
6193             "0.95"
6194           ],
6195           "value": [
6196             "0.95"
6197           ]
6198         },
6199         "datasource": {
6200           "type": "prometheus",
6201           "uid": "PBFA97CFB590B2093"
6202         },
6203         "definition": "label_values(quantile)",
6204         "hide": 0,
6205         "includeAll": true,
6206         "label": "Percentile",
6207         "multi": true,
6208         "name": "percentile",
6209         "options": [],
6210         "query": {
6211           "query": "label_values(quantile)",
6212           "refId": "Prometheus-percentile-Variable-Query"
6213         },
6214         "refresh": 1,
6215         "regex": "",
6216         "skipUrlSync": false,
6217         "sort": 0,
6218         "tagValuesQuery": "",
6219         "tagsQuery": "",
6220         "type": "query",
6221         "useTags": false
6222       }
6223     ]
6224   },
6225   "time": {
6226     "from": "now-5m",
6227     "to": "now"
6228   },
6229   "timepicker": {
6230     "refresh_intervals": [
6231       "5s",
6232       "10s",
6233       "30s",
6234       "1m",
6235       "5m",
6236       "15m",
6237       "30m",
6238       "1h",
6239       "2h",
6240       "1d"
6241     ],
6242     "time_options": [
6243       "5m",
6244       "15m",
6245       "1h",
6246       "6h",
6247       "12h",
6248       "24h",
6249       "2d",
6250       "7d",
6251       "30d"
6252     ]
6253   },
6254   "timezone": "browser",
6255   "title": "Kafka cluster",
6256   "uid": "qu-QZdfZz",
6257   "version": 1,
6258   "weekStart": ""
6259 }