juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of active controllers in the cluster.",
  64       "fieldConfig": {
  65         "defaults": {
  66           "mappings": [
  67             {
  68               "options": {
  69                 "match": "null",
  70                 "result": {
  71                   "text": "N/A"
  72                 }
  73               },
  74               "type": "special"
  75             }
  76           ],
  77           "thresholds": {
  78             "mode": "absolute",
  79             "steps": [
  80               {
  81                 "color": "#299c46",
  82                 "value": null
  83               },
  84               {
  85                 "color": "#e5ac0e",
  86                 "value": 2
  87               },
  88               {
  89                 "color": "#bf1b00"
  90               }
  91             ]
  92           },
  93           "unit": "none"
  94         },
  95         "overrides": []
  96       },
  97       "gridPos": {
  98         "h": 4,
  99         "w": 4,
 100         "x": 0,
 101         "y": 1
 102       },
 103       "id": 12,
 104       "links": [],
 105       "maxDataPoints": 100,
 106       "options": {
 107         "colorMode": "value",
 108         "graphMode": "none",
 109         "justifyMode": "auto",
 110         "orientation": "vertical",
 111         "reduceOptions": {
 112           "calcs": [
 113             "lastNotNull"
 114           ],
 115           "fields": "",
 116           "values": false
 117         },
 118         "text": {},
 119         "textMode": "value_and_name"
 120       },
 121       "pluginVersion": "10.0.3",
 122       "targets": [
 123         {
 124           "datasource": {
 125             "type": "prometheus",
 126             "uid": "PBFA97CFB590B2093"
 127           },
 128           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 129           "format": "time_series",
 130           "instant": true,
 131           "interval": "",
 132           "intervalFactor": 1,
 133           "legendFormat": "{{instance}}",
 134           "refId": "A"
 135         }
 136       ],
 137       "title": "Active Controllers",
 138       "type": "stat"
 139     },
 140     {
 141       "datasource": {
 142         "type": "prometheus",
 143         "uid": "PBFA97CFB590B2093"
 144       },
 145       "description": "Number of Brokers Online",
 146       "fieldConfig": {
 147         "defaults": {
 148           "color": {
 149             "mode": "thresholds"
 150           },
 151           "mappings": [
 152             {
 153               "options": {
 154                 "match": "null",
 155                 "result": {
 156                   "text": "N/A"
 157                 }
 158               },
 159               "type": "special"
 160             }
 161           ],
 162           "thresholds": {
 163             "mode": "absolute",
 164             "steps": [
 165               {
 166                 "color": "#d44a3a",
 167                 "value": null
 168               },
 169               {
 170                 "color": "rgba(237, 129, 40, 0.89)",
 171                 "value": 0
 172               },
 173               {
 174                 "color": "#299c46",
 175                 "value": 2
 176               }
 177             ]
 178           },
 179           "unit": "none"
 180         },
 181         "overrides": []
 182       },
 183       "gridPos": {
 184         "h": 4,
 185         "w": 4,
 186         "x": 4,
 187         "y": 1
 188       },
 189       "id": 14,
 190       "links": [],
 191       "maxDataPoints": 100,
 192       "options": {
 193         "colorMode": "value",
 194         "graphMode": "none",
 195         "justifyMode": "auto",
 196         "orientation": "horizontal",
 197         "reduceOptions": {
 198           "calcs": [
 199             "lastNotNull"
 200           ],
 201           "fields": "",
 202           "values": false
 203         },
 204         "text": {},
 205         "textMode": "auto"
 206       },
 207       "pluginVersion": "10.0.3",
 208       "repeatDirection": "h",
 209       "targets": [
 210         {
 211           "datasource": {
 212             "type": "prometheus",
 213             "uid": "PBFA97CFB590B2093"
 214           },
 215           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 216           "format": "time_series",
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 1,
 220           "legendFormat": "",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Brokers Online",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Partitions that are online",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#d44a3a",
 254                 "value": null
 255               },
 256               {
 257                 "color": "rgba(237, 129, 40, 0.89)",
 258                 "value": 0
 259               },
 260               {
 261                 "color": "#299c46",
 262                 "value": 0
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 4,
 273         "x": 8,
 274         "y": 1
 275       },
 276       "id": 18,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Online Partitions",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 15,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Unclean leader election rate",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#299c46",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 2
 430               },
 431               {
 432                 "color": "#d44a3a"
 433               }
 434             ]
 435           },
 436           "unit": "none"
 437         },
 438         "overrides": []
 439       },
 440       "gridPos": {
 441         "h": 4,
 442         "w": 4,
 443         "x": 19,
 444         "y": 1
 445       },
 446       "id": 16,
 447       "links": [],
 448       "maxDataPoints": 100,
 449       "options": {
 450         "colorMode": "value",
 451         "graphMode": "none",
 452         "justifyMode": "auto",
 453         "orientation": "horizontal",
 454         "reduceOptions": {
 455           "calcs": [
 456             "lastNotNull"
 457           ],
 458           "fields": "",
 459           "values": false
 460         },
 461         "text": {},
 462         "textMode": "auto"
 463       },
 464       "pluginVersion": "10.0.3",
 465       "targets": [
 466         {
 467           "datasource": {
 468             "type": "prometheus",
 469             "uid": "PBFA97CFB590B2093"
 470           },
 471           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 472           "format": "time_series",
 473           "instant": true,
 474           "interval": "",
 475           "intervalFactor": 1,
 476           "legendFormat": "",
 477           "refId": "A"
 478         }
 479       ],
 480       "title": "Unclean Leader Election Rate",
 481       "type": "stat"
 482     },
 483     {
 484       "datasource": {
 485         "type": "prometheus",
 486         "uid": "PBFA97CFB590B2093"
 487       },
 488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 489       "fieldConfig": {
 490         "defaults": {
 491           "color": {
 492             "mode": "thresholds"
 493           },
 494           "mappings": [
 495             {
 496               "options": {
 497                 "match": "null",
 498                 "result": {
 499                   "text": "N/A"
 500                 }
 501               },
 502               "type": "special"
 503             }
 504           ],
 505           "thresholds": {
 506             "mode": "absolute",
 507             "steps": [
 508               {
 509                 "color": "#508642",
 510                 "value": null
 511               },
 512               {
 513                 "color": "rgba(237, 129, 40, 0.89)",
 514                 "value": 1
 515               },
 516               {
 517                 "color": "#bf1b00",
 518                 "value": 5
 519               }
 520             ]
 521           },
 522           "unit": "none"
 523         },
 524         "overrides": []
 525       },
 526       "gridPos": {
 527         "h": 4,
 528         "w": 4,
 529         "x": 0,
 530         "y": 5
 531       },
 532       "id": 20,
 533       "links": [],
 534       "maxDataPoints": 100,
 535       "options": {
 536         "colorMode": "value",
 537         "graphMode": "none",
 538         "justifyMode": "auto",
 539         "orientation": "horizontal",
 540         "reduceOptions": {
 541           "calcs": [
 542             "lastNotNull"
 543           ],
 544           "fields": "",
 545           "values": false
 546         },
 547         "text": {},
 548         "textMode": "auto"
 549       },
 550       "pluginVersion": "10.0.3",
 551       "targets": [
 552         {
 553           "datasource": {
 554             "type": "prometheus",
 555             "uid": "PBFA97CFB590B2093"
 556           },
 557           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 558           "format": "time_series",
 559           "hide": false,
 560           "instant": true,
 561           "interval": "",
 562           "intervalFactor": 2,
 563           "legendFormat": "",
 564           "refId": "A"
 565         }
 566       ],
 567       "title": "Under Replicated Partitions",
 568       "type": "stat"
 569     },
 570     {
 571       "datasource": {
 572         "type": "prometheus",
 573         "uid": "PBFA97CFB590B2093"
 574       },
 575       "description": "",
 576       "fieldConfig": {
 577         "defaults": {
 578           "color": {
 579             "mode": "palette-classic"
 580           },
 581           "custom": {
 582             "axisCenteredZero": false,
 583             "axisColorMode": "text",
 584             "axisLabel": "",
 585             "axisPlacement": "auto",
 586             "barAlignment": 0,
 587             "drawStyle": "line",
 588             "fillOpacity": 10,
 589             "gradientMode": "none",
 590             "hideFrom": {
 591               "legend": false,
 592               "tooltip": false,
 593               "viz": false
 594             },
 595             "lineInterpolation": "linear",
 596             "lineWidth": 1,
 597             "pointSize": 5,
 598             "scaleDistribution": {
 599               "type": "linear"
 600             },
 601             "showPoints": "never",
 602             "spanNulls": true,
 603             "stacking": {
 604               "group": "A",
 605               "mode": "none"
 606             },
 607             "thresholdsStyle": {
 608               "mode": "off"
 609             }
 610           },
 611           "decimals": 0,
 612           "links": [],
 613           "mappings": [],
 614           "min": 0,
 615           "thresholds": {
 616             "mode": "absolute",
 617             "steps": [
 618               {
 619                 "color": "green",
 620                 "value": null
 621               },
 622               {
 623                 "color": "red",
 624                 "value": 80
 625               }
 626             ]
 627           },
 628           "unit": "partitions"
 629         },
 630         "overrides": []
 631       },
 632       "gridPos": {
 633         "h": 12,
 634         "w": 11,
 635         "x": 4,
 636         "y": 5
 637       },
 638       "id": 123,
 639       "options": {
 640         "legend": {
 641           "calcs": [
 642             "lastNotNull",
 643             "max"
 644           ],
 645           "displayMode": "table",
 646           "placement": "bottom",
 647           "showLegend": true
 648         },
 649         "tooltip": {
 650           "mode": "single",
 651           "sort": "none"
 652         }
 653       },
 654       "pluginVersion": "8.1.3",
 655       "targets": [
 656         {
 657           "datasource": {
 658             "type": "prometheus",
 659             "uid": "PBFA97CFB590B2093"
 660           },
 661           "editorMode": "code",
 662           "exemplar": false,
 663           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 664           "hide": false,
 665           "interval": "",
 666           "legendFormat": "{{instance}}",
 667           "range": true,
 668           "refId": "A"
 669         }
 670       ],
 671       "title": "Underreplicated Partitions",
 672       "type": "timeseries"
 673     },
 674     {
 675       "datasource": {
 676         "type": "prometheus",
 677         "uid": "PBFA97CFB590B2093"
 678       },
 679       "description": "",
 680       "fieldConfig": {
 681         "defaults": {
 682           "color": {
 683             "mode": "palette-classic"
 684           },
 685           "custom": {
 686             "axisCenteredZero": false,
 687             "axisColorMode": "text",
 688             "axisLabel": "",
 689             "axisPlacement": "auto",
 690             "barAlignment": 0,
 691             "drawStyle": "line",
 692             "fillOpacity": 10,
 693             "gradientMode": "none",
 694             "hideFrom": {
 695               "legend": false,
 696               "tooltip": false,
 697               "viz": false
 698             },
 699             "lineInterpolation": "linear",
 700             "lineWidth": 1,
 701             "pointSize": 5,
 702             "scaleDistribution": {
 703               "type": "linear"
 704             },
 705             "showPoints": "never",
 706             "spanNulls": true,
 707             "stacking": {
 708               "group": "A",
 709               "mode": "none"
 710             },
 711             "thresholdsStyle": {
 712               "mode": "off"
 713             }
 714           },
 715           "decimals": 0,
 716           "links": [],
 717           "mappings": [
 718             {
 719               "options": {
 720                 "null": {
 721                   "index": 0,
 722                   "text": "0"
 723                 }
 724               },
 725               "type": "value"
 726             }
 727           ],
 728           "min": 0,
 729           "thresholds": {
 730             "mode": "absolute",
 731             "steps": [
 732               {
 733                 "color": "green",
 734                 "value": null
 735               },
 736               {
 737                 "color": "red",
 738                 "value": 80
 739               }
 740             ]
 741           },
 742           "unit": "partitions"
 743         },
 744         "overrides": []
 745       },
 746       "gridPos": {
 747         "h": 12,
 748         "w": 8,
 749         "x": 15,
 750         "y": 5
 751       },
 752       "id": 124,
 753       "options": {
 754         "legend": {
 755           "calcs": [
 756             "lastNotNull",
 757             "max"
 758           ],
 759           "displayMode": "table",
 760           "placement": "bottom",
 761           "showLegend": true
 762         },
 763         "tooltip": {
 764           "mode": "single",
 765           "sort": "none"
 766         }
 767       },
 768       "pluginVersion": "8.1.3",
 769       "targets": [
 770         {
 771           "datasource": {
 772             "type": "prometheus",
 773             "uid": "PBFA97CFB590B2093"
 774           },
 775           "editorMode": "code",
 776           "exemplar": false,
 777           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 778           "hide": false,
 779           "interval": "",
 780           "legendFormat": "{{instance}}",
 781           "range": true,
 782           "refId": "A"
 783         }
 784       ],
 785       "title": "Leader Count",
 786       "type": "timeseries"
 787     },
 788     {
 789       "datasource": {
 790         "type": "prometheus",
 791         "uid": "PBFA97CFB590B2093"
 792       },
 793       "description": "Number of partitions under min insync replicas.",
 794       "fieldConfig": {
 795         "defaults": {
 796           "color": {
 797             "mode": "thresholds"
 798           },
 799           "mappings": [
 800             {
 801               "options": {
 802                 "match": "null",
 803                 "result": {
 804                   "text": "N/A"
 805                 }
 806               },
 807               "type": "special"
 808             }
 809           ],
 810           "thresholds": {
 811             "mode": "absolute",
 812             "steps": [
 813               {
 814                 "color": "#508642",
 815                 "value": null
 816               },
 817               {
 818                 "color": "rgba(237, 129, 40, 0.89)",
 819                 "value": 1
 820               },
 821               {
 822                 "color": "#bf1b00",
 823                 "value": 5
 824               }
 825             ]
 826           },
 827           "unit": "none"
 828         },
 829         "overrides": []
 830       },
 831       "gridPos": {
 832         "h": 4,
 833         "w": 4,
 834         "x": 0,
 835         "y": 9
 836       },
 837       "id": 32,
 838       "links": [],
 839       "maxDataPoints": 100,
 840       "options": {
 841         "colorMode": "value",
 842         "graphMode": "none",
 843         "justifyMode": "auto",
 844         "orientation": "horizontal",
 845         "reduceOptions": {
 846           "calcs": [
 847             "lastNotNull"
 848           ],
 849           "fields": "",
 850           "values": false
 851         },
 852         "text": {},
 853         "textMode": "auto"
 854       },
 855       "pluginVersion": "10.0.3",
 856       "targets": [
 857         {
 858           "datasource": {
 859             "type": "prometheus",
 860             "uid": "PBFA97CFB590B2093"
 861           },
 862           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 863           "format": "time_series",
 864           "hide": false,
 865           "instant": true,
 866           "interval": "",
 867           "intervalFactor": 2,
 868           "legendFormat": "",
 869           "refId": "A"
 870         }
 871       ],
 872       "title": "Under Min ISR Partitions",
 873       "type": "stat"
 874     },
 875     {
 876       "datasource": {
 877         "type": "prometheus",
 878         "uid": "PBFA97CFB590B2093"
 879       },
 880       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 881       "fieldConfig": {
 882         "defaults": {
 883           "color": {
 884             "mode": "thresholds"
 885           },
 886           "mappings": [
 887             {
 888               "options": {
 889                 "match": "null",
 890                 "result": {
 891                   "text": "N/A"
 892                 }
 893               },
 894               "type": "special"
 895             }
 896           ],
 897           "thresholds": {
 898             "mode": "absolute",
 899             "steps": [
 900               {
 901                 "color": "#508642",
 902                 "value": null
 903               },
 904               {
 905                 "color": "#ef843c",
 906                 "value": 1
 907               },
 908               {
 909                 "color": "#bf1b00",
 910                 "value": 1
 911               }
 912             ]
 913           },
 914           "unit": "none"
 915         },
 916         "overrides": []
 917       },
 918       "gridPos": {
 919         "h": 4,
 920         "w": 4,
 921         "x": 0,
 922         "y": 13
 923       },
 924       "id": 22,
 925       "links": [],
 926       "maxDataPoints": 100,
 927       "options": {
 928         "colorMode": "value",
 929         "graphMode": "none",
 930         "justifyMode": "auto",
 931         "orientation": "horizontal",
 932         "reduceOptions": {
 933           "calcs": [
 934             "lastNotNull"
 935           ],
 936           "fields": "",
 937           "values": false
 938         },
 939         "text": {},
 940         "textMode": "auto"
 941       },
 942       "pluginVersion": "10.0.3",
 943       "targets": [
 944         {
 945           "datasource": {
 946             "type": "prometheus",
 947             "uid": "PBFA97CFB590B2093"
 948           },
 949           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 950           "format": "time_series",
 951           "instant": true,
 952           "interval": "",
 953           "intervalFactor": 1,
 954           "legendFormat": "",
 955           "refId": "A"
 956         }
 957       ],
 958       "title": "Offline Partitions Count",
 959       "type": "stat"
 960     },
 961     {
 962       "collapsed": true,
 963       "datasource": {
 964         "type": "datasource",
 965         "uid": "grafana"
 966       },
 967       "gridPos": {
 968         "h": 1,
 969         "w": 24,
 970         "x": 0,
 971         "y": 17
 972       },
 973       "id": 40,
 974       "panels": [
 975         {
 976           "datasource": {
 977             "type": "prometheus",
 978             "uid": "PBFA97CFB590B2093"
 979           },
 980           "fieldConfig": {
 981             "defaults": {
 982               "color": {
 983                 "mode": "palette-classic"
 984               },
 985               "custom": {
 986                 "axisCenteredZero": false,
 987                 "axisColorMode": "text",
 988                 "axisLabel": "Cores",
 989                 "axisPlacement": "auto",
 990                 "barAlignment": 0,
 991                 "drawStyle": "line",
 992                 "fillOpacity": 10,
 993                 "gradientMode": "none",
 994                 "hideFrom": {
 995                   "legend": false,
 996                   "tooltip": false,
 997                   "viz": false
 998                 },
 999                 "lineInterpolation": "linear",
1000                 "lineWidth": 2,
1001                 "pointSize": 5,
1002                 "scaleDistribution": {
1003                   "type": "linear"
1004                 },
1005                 "showPoints": "never",
1006                 "spanNulls": false,
1007                 "stacking": {
1008                   "group": "A",
1009                   "mode": "none"
1010                 },
1011                 "thresholdsStyle": {
1012                   "mode": "off"
1013                 }
1014               },
1015               "links": [],
1016               "mappings": [],
1017               "thresholds": {
1018                 "mode": "absolute",
1019                 "steps": [
1020                   {
1021                     "color": "green",
1022                     "value": null
1023                   },
1024                   {
1025                     "color": "red",
1026                     "value": 80
1027                   }
1028                 ]
1029               },
1030               "unit": "percent"
1031             },
1032             "overrides": [
1033               {
1034                 "matcher": {
1035                   "id": "byName",
1036                   "options": "localhost:7071"
1037                 },
1038                 "properties": [
1039                   {
1040                     "id": "color",
1041                     "value": {
1042                       "fixedColor": "#629E51",
1043                       "mode": "fixed"
1044                     }
1045                   }
1046                 ]
1047               }
1048             ]
1049           },
1050           "gridPos": {
1051             "h": 12,
1052             "w": 8,
1053             "x": 0,
1054             "y": 19
1055           },
1056           "id": 27,
1057           "links": [],
1058           "options": {
1059             "legend": {
1060               "calcs": [
1061                 "mean",
1062                 "lastNotNull",
1063                 "max"
1064               ],
1065               "displayMode": "table",
1066               "placement": "bottom",
1067               "showLegend": true
1068             },
1069             "tooltip": {
1070               "mode": "single",
1071               "sort": "none"
1072             }
1073           },
1074           "pluginVersion": "8.1.3",
1075           "targets": [
1076             {
1077               "datasource": {
1078                 "type": "prometheus",
1079                 "uid": "PBFA97CFB590B2093"
1080               },
1081               "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1082               "format": "time_series",
1083               "interval": "",
1084               "intervalFactor": 2,
1085               "legendFormat": "{{instance}}",
1086               "metric": "process_cpu_secondspersec",
1087               "refId": "A",
1088               "step": 4
1089             }
1090           ],
1091           "title": "CPU Usage",
1092           "type": "timeseries"
1093         },
1094         {
1095           "datasource": {
1096             "type": "prometheus",
1097             "uid": "PBFA97CFB590B2093"
1098           },
1099           "fieldConfig": {
1100             "defaults": {
1101               "color": {
1102                 "mode": "palette-classic"
1103               },
1104               "custom": {
1105                 "axisCenteredZero": false,
1106                 "axisColorMode": "text",
1107                 "axisLabel": "Memory",
1108                 "axisPlacement": "auto",
1109                 "barAlignment": 0,
1110                 "drawStyle": "line",
1111                 "fillOpacity": 10,
1112                 "gradientMode": "none",
1113                 "hideFrom": {
1114                   "legend": false,
1115                   "tooltip": false,
1116                   "viz": false
1117                 },
1118                 "lineInterpolation": "linear",
1119                 "lineWidth": 2,
1120                 "pointSize": 5,
1121                 "scaleDistribution": {
1122                   "type": "linear"
1123                 },
1124                 "showPoints": "never",
1125                 "spanNulls": false,
1126                 "stacking": {
1127                   "group": "A",
1128                   "mode": "none"
1129                 },
1130                 "thresholdsStyle": {
1131                   "mode": "off"
1132                 }
1133               },
1134               "links": [],
1135               "mappings": [],
1136               "thresholds": {
1137                 "mode": "absolute",
1138                 "steps": [
1139                   {
1140                     "color": "green",
1141                     "value": null
1142                   },
1143                   {
1144                     "color": "red",
1145                     "value": 80
1146                   }
1147                 ]
1148               },
1149               "unit": "bytes"
1150             },
1151             "overrides": [
1152               {
1153                 "matcher": {
1154                   "id": "byName",
1155                   "options": "localhost:7071"
1156                 },
1157                 "properties": [
1158                   {
1159                     "id": "color",
1160                     "value": {
1161                       "fixedColor": "#BA43A9",
1162                       "mode": "fixed"
1163                     }
1164                   }
1165                 ]
1166               },
1167               {
1168                 "__systemRef": "hideSeriesFrom",
1169                 "matcher": {
1170                   "id": "byNames",
1171                   "options": {
1172                     "mode": "exclude",
1173                     "names": [
1174                       "kafka-2:80"
1175                     ],
1176                     "prefix": "All except:",
1177                     "readOnly": true
1178                   }
1179                 },
1180                 "properties": [
1181                   {
1182                     "id": "custom.hideFrom",
1183                     "value": {
1184                       "legend": false,
1185                       "tooltip": false,
1186                       "viz": true
1187                     }
1188                   }
1189                 ]
1190               }
1191             ]
1192           },
1193           "gridPos": {
1194             "h": 12,
1195             "w": 8,
1196             "x": 8,
1197             "y": 19
1198           },
1199           "id": 2,
1200           "links": [],
1201           "options": {
1202             "legend": {
1203               "calcs": [
1204                 "mean",
1205                 "lastNotNull",
1206                 "max"
1207               ],
1208               "displayMode": "table",
1209               "placement": "bottom",
1210               "showLegend": true
1211             },
1212             "tooltip": {
1213               "mode": "single",
1214               "sort": "none"
1215             }
1216           },
1217           "pluginVersion": "8.1.3",
1218           "targets": [
1219             {
1220               "datasource": {
1221                 "type": "prometheus",
1222                 "uid": "PBFA97CFB590B2093"
1223               },
1224               "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1225               "interval": "",
1226               "intervalFactor": 2,
1227               "legendFormat": "{{instance}}",
1228               "metric": "jvm_memory_bytes_used",
1229               "refId": "A",
1230               "step": 4
1231             },
1232             {
1233               "datasource": {
1234                 "type": "prometheus",
1235                 "uid": "PBFA97CFB590B2093"
1236               },
1237               "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1238               "interval": "",
1239               "legendFormat": "{{instance}}",
1240               "refId": "B"
1241             }
1242           ],
1243           "title": "JVM Memory Used",
1244           "type": "timeseries"
1245         },
1246         {
1247           "datasource": {
1248             "type": "prometheus",
1249             "uid": "PBFA97CFB590B2093"
1250           },
1251           "fieldConfig": {
1252             "defaults": {
1253               "color": {
1254                 "mode": "palette-classic"
1255               },
1256               "custom": {
1257                 "axisCenteredZero": false,
1258                 "axisColorMode": "text",
1259                 "axisLabel": "% time in GC",
1260                 "axisPlacement": "auto",
1261                 "barAlignment": 0,
1262                 "drawStyle": "line",
1263                 "fillOpacity": 10,
1264                 "gradientMode": "none",
1265                 "hideFrom": {
1266                   "legend": false,
1267                   "tooltip": false,
1268                   "viz": false
1269                 },
1270                 "lineInterpolation": "linear",
1271                 "lineWidth": 2,
1272                 "pointSize": 5,
1273                 "scaleDistribution": {
1274                   "type": "linear"
1275                 },
1276                 "showPoints": "never",
1277                 "spanNulls": false,
1278                 "stacking": {
1279                   "group": "A",
1280                   "mode": "none"
1281                 },
1282                 "thresholdsStyle": {
1283                   "mode": "off"
1284                 }
1285               },
1286               "links": [],
1287               "mappings": [],
1288               "thresholds": {
1289                 "mode": "absolute",
1290                 "steps": [
1291                   {
1292                     "color": "green",
1293                     "value": null
1294                   },
1295                   {
1296                     "color": "red",
1297                     "value": 80
1298                   }
1299                 ]
1300               },
1301               "unit": "percentunit"
1302             },
1303             "overrides": [
1304               {
1305                 "matcher": {
1306                   "id": "byName",
1307                   "options": "localhost:7071"
1308                 },
1309                 "properties": [
1310                   {
1311                     "id": "color",
1312                     "value": {
1313                       "fixedColor": "#890F02",
1314                       "mode": "fixed"
1315                     }
1316                   }
1317                 ]
1318               }
1319             ]
1320           },
1321           "gridPos": {
1322             "h": 12,
1323             "w": 8,
1324             "x": 16,
1325             "y": 19
1326           },
1327           "id": 3,
1328           "links": [],
1329           "options": {
1330             "legend": {
1331               "calcs": [
1332                 "mean",
1333                 "lastNotNull",
1334                 "max"
1335               ],
1336               "displayMode": "table",
1337               "placement": "bottom",
1338               "showLegend": true
1339             },
1340             "tooltip": {
1341               "mode": "single",
1342               "sort": "none"
1343             }
1344           },
1345           "pluginVersion": "8.1.3",
1346           "targets": [
1347             {
1348               "datasource": {
1349                 "type": "prometheus",
1350                 "uid": "PBFA97CFB590B2093"
1351               },
1352               "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1353               "interval": "",
1354               "intervalFactor": 2,
1355               "legendFormat": "{{instance}}",
1356               "metric": "jvm_gc_collection_seconds_sum",
1357               "refId": "A",
1358               "step": 4
1359             }
1360           ],
1361           "title": "Time spent in GC",
1362           "type": "timeseries"
1363         }
1364       ],
1365       "targets": [
1366         {
1367           "datasource": {
1368             "type": "datasource",
1369             "uid": "grafana"
1370           },
1371           "refId": "A"
1372         }
1373       ],
1374       "title": "System",
1375       "type": "row"
1376     },
1377     {
1378       "collapsed": false,
1379       "datasource": {
1380         "type": "datasource",
1381         "uid": "grafana"
1382       },
1383       "gridPos": {
1384         "h": 1,
1385         "w": 24,
1386         "x": 0,
1387         "y": 18
1388       },
1389       "id": 31,
1390       "panels": [],
1391       "targets": [
1392         {
1393           "datasource": {
1394             "type": "datasource",
1395             "uid": "grafana"
1396           },
1397           "refId": "A"
1398         }
1399       ],
1400       "title": "Throughput & Request rate",
1401       "type": "row"
1402     },
1403     {
1404       "datasource": {
1405         "type": "prometheus",
1406         "uid": "PBFA97CFB590B2093"
1407       },
1408       "fieldConfig": {
1409         "defaults": {
1410           "color": {
1411             "mode": "palette-classic"
1412           },
1413           "custom": {
1414             "axisCenteredZero": false,
1415             "axisColorMode": "text",
1416             "axisLabel": "Bytes/s",
1417             "axisPlacement": "auto",
1418             "barAlignment": 0,
1419             "drawStyle": "line",
1420             "fillOpacity": 10,
1421             "gradientMode": "none",
1422             "hideFrom": {
1423               "legend": false,
1424               "tooltip": false,
1425               "viz": false
1426             },
1427             "lineInterpolation": "linear",
1428             "lineWidth": 2,
1429             "pointSize": 5,
1430             "scaleDistribution": {
1431               "type": "linear"
1432             },
1433             "showPoints": "never",
1434             "spanNulls": false,
1435             "stacking": {
1436               "group": "A",
1437               "mode": "none"
1438             },
1439             "thresholdsStyle": {
1440               "mode": "off"
1441             }
1442           },
1443           "links": [],
1444           "mappings": [],
1445           "thresholds": {
1446             "mode": "absolute",
1447             "steps": [
1448               {
1449                 "color": "green",
1450                 "value": null
1451               },
1452               {
1453                 "color": "red",
1454                 "value": 80
1455               }
1456             ]
1457           },
1458           "unit": "Bps"
1459         },
1460         "overrides": []
1461       },
1462       "gridPos": {
1463         "h": 8,
1464         "w": 12,
1465         "x": 0,
1466         "y": 19
1467       },
1468       "id": 84,
1469       "links": [],
1470       "options": {
1471         "legend": {
1472           "calcs": [
1473             "mean",
1474             "lastNotNull",
1475             "max"
1476           ],
1477           "displayMode": "list",
1478           "placement": "bottom",
1479           "showLegend": true
1480         },
1481         "tooltip": {
1482           "mode": "single",
1483           "sort": "none"
1484         }
1485       },
1486       "pluginVersion": "8.1.3",
1487       "targets": [
1488         {
1489           "datasource": {
1490             "type": "prometheus",
1491             "uid": "PBFA97CFB590B2093"
1492           },
1493           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1494           "format": "time_series",
1495           "interval": "",
1496           "intervalFactor": 2,
1497           "legendFormat": "Bytes in",
1498           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1499           "refId": "A",
1500           "step": 4
1501         },
1502         {
1503           "datasource": {
1504             "type": "prometheus",
1505             "uid": "PBFA97CFB590B2093"
1506           },
1507           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1508           "format": "time_series",
1509           "hide": false,
1510           "interval": "",
1511           "intervalFactor": 2,
1512           "legendFormat": "Bytes out",
1513           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1514           "refId": "B",
1515           "step": 4
1516         }
1517       ],
1518       "title": "Broker network throughput",
1519       "type": "timeseries"
1520     },
1521     {
1522       "datasource": {
1523         "type": "prometheus",
1524         "uid": "PBFA97CFB590B2093"
1525       },
1526       "description": "Produce request rate.",
1527       "fieldConfig": {
1528         "defaults": {
1529           "mappings": [
1530             {
1531               "options": {
1532                 "match": "null",
1533                 "result": {
1534                   "text": "N/A"
1535                 }
1536               },
1537               "type": "special"
1538             }
1539           ],
1540           "thresholds": {
1541             "mode": "absolute",
1542             "steps": [
1543               {
1544                 "color": "green",
1545                 "value": null
1546               }
1547             ]
1548           },
1549           "unit": "none"
1550         },
1551         "overrides": []
1552       },
1553       "gridPos": {
1554         "h": 4,
1555         "w": 4,
1556         "x": 12,
1557         "y": 19
1558       },
1559       "id": 93,
1560       "links": [],
1561       "maxDataPoints": 100,
1562       "options": {
1563         "colorMode": "value",
1564         "graphMode": "area",
1565         "justifyMode": "auto",
1566         "orientation": "horizontal",
1567         "reduceOptions": {
1568           "calcs": [
1569             "lastNotNull"
1570           ],
1571           "fields": "",
1572           "values": false
1573         },
1574         "text": {},
1575         "textMode": "auto"
1576       },
1577       "pluginVersion": "10.0.3",
1578       "targets": [
1579         {
1580           "datasource": {
1581             "type": "prometheus",
1582             "uid": "PBFA97CFB590B2093"
1583           },
1584           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1585           "interval": "",
1586           "legendFormat": "",
1587           "refId": "A"
1588         }
1589       ],
1590       "title": "All Request Per Sec",
1591       "type": "stat"
1592     },
1593     {
1594       "datasource": {
1595         "type": "prometheus",
1596         "uid": "PBFA97CFB590B2093"
1597       },
1598       "description": "Fetch request rate.",
1599       "fieldConfig": {
1600         "defaults": {
1601           "mappings": [
1602             {
1603               "options": {
1604                 "match": "null",
1605                 "result": {
1606                   "text": "N/A"
1607                 }
1608               },
1609               "type": "special"
1610             }
1611           ],
1612           "thresholds": {
1613             "mode": "absolute",
1614             "steps": [
1615               {
1616                 "color": "green",
1617                 "value": null
1618               }
1619             ]
1620           },
1621           "unit": "none"
1622         },
1623         "overrides": []
1624       },
1625       "gridPos": {
1626         "h": 4,
1627         "w": 4,
1628         "x": 16,
1629         "y": 19
1630       },
1631       "id": 37,
1632       "links": [],
1633       "maxDataPoints": 100,
1634       "options": {
1635         "colorMode": "value",
1636         "graphMode": "area",
1637         "justifyMode": "auto",
1638         "orientation": "horizontal",
1639         "reduceOptions": {
1640           "calcs": [
1641             "lastNotNull"
1642           ],
1643           "fields": "",
1644           "values": false
1645         },
1646         "text": {},
1647         "textMode": "auto"
1648       },
1649       "pluginVersion": "10.0.3",
1650       "targets": [
1651         {
1652           "datasource": {
1653             "type": "prometheus",
1654             "uid": "PBFA97CFB590B2093"
1655           },
1656           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1657           "interval": "",
1658           "legendFormat": "",
1659           "refId": "A"
1660         }
1661       ],
1662       "title": "Consumer Fetch Request Per Sec",
1663       "type": "stat"
1664     },
1665     {
1666       "datasource": {
1667         "type": "prometheus",
1668         "uid": "PBFA97CFB590B2093"
1669       },
1670       "description": "Produce request rate.",
1671       "fieldConfig": {
1672         "defaults": {
1673           "mappings": [
1674             {
1675               "options": {
1676                 "match": "null",
1677                 "result": {
1678                   "text": "N/A"
1679                 }
1680               },
1681               "type": "special"
1682             }
1683           ],
1684           "thresholds": {
1685             "mode": "absolute",
1686             "steps": [
1687               {
1688                 "color": "green",
1689                 "value": null
1690               }
1691             ]
1692           },
1693           "unit": "none"
1694         },
1695         "overrides": []
1696       },
1697       "gridPos": {
1698         "h": 4,
1699         "w": 4,
1700         "x": 20,
1701         "y": 19
1702       },
1703       "id": 35,
1704       "links": [],
1705       "maxDataPoints": 100,
1706       "options": {
1707         "colorMode": "value",
1708         "graphMode": "area",
1709         "justifyMode": "auto",
1710         "orientation": "horizontal",
1711         "reduceOptions": {
1712           "calcs": [
1713             "lastNotNull"
1714           ],
1715           "fields": "",
1716           "values": false
1717         },
1718         "text": {},
1719         "textMode": "auto"
1720       },
1721       "pluginVersion": "10.0.3",
1722       "targets": [
1723         {
1724           "datasource": {
1725             "type": "prometheus",
1726             "uid": "PBFA97CFB590B2093"
1727           },
1728           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1729           "interval": "",
1730           "legendFormat": "",
1731           "refId": "A"
1732         }
1733       ],
1734       "title": "Produce Request Per Sec",
1735       "type": "stat"
1736     },
1737     {
1738       "datasource": {
1739         "type": "prometheus",
1740         "uid": "PBFA97CFB590B2093"
1741       },
1742       "description": "Metadata request rate.",
1743       "fieldConfig": {
1744         "defaults": {
1745           "mappings": [
1746             {
1747               "options": {
1748                 "match": "null",
1749                 "result": {
1750                   "text": "N/A"
1751                 }
1752               },
1753               "type": "special"
1754             }
1755           ],
1756           "thresholds": {
1757             "mode": "absolute",
1758             "steps": [
1759               {
1760                 "color": "green",
1761                 "value": null
1762               }
1763             ]
1764           },
1765           "unit": "none"
1766         },
1767         "overrides": []
1768       },
1769       "gridPos": {
1770         "h": 4,
1771         "w": 4,
1772         "x": 12,
1773         "y": 23
1774       },
1775       "id": 36,
1776       "links": [],
1777       "maxDataPoints": 100,
1778       "options": {
1779         "colorMode": "value",
1780         "graphMode": "area",
1781         "justifyMode": "auto",
1782         "orientation": "horizontal",
1783         "reduceOptions": {
1784           "calcs": [
1785             "lastNotNull"
1786           ],
1787           "fields": "",
1788           "values": false
1789         },
1790         "text": {},
1791         "textMode": "auto"
1792       },
1793       "pluginVersion": "10.0.3",
1794       "targets": [
1795         {
1796           "datasource": {
1797             "type": "prometheus",
1798             "uid": "PBFA97CFB590B2093"
1799           },
1800           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1801           "interval": "",
1802           "legendFormat": "",
1803           "refId": "A"
1804         }
1805       ],
1806       "title": "Metadata Request Per Sec",
1807       "type": "stat"
1808     },
1809     {
1810       "datasource": {
1811         "type": "prometheus",
1812         "uid": "PBFA97CFB590B2093"
1813       },
1814       "description": "Offset Commit request rate.",
1815       "fieldConfig": {
1816         "defaults": {
1817           "mappings": [
1818             {
1819               "options": {
1820                 "match": "null",
1821                 "result": {
1822                   "text": "N/A"
1823                 }
1824               },
1825               "type": "special"
1826             }
1827           ],
1828           "thresholds": {
1829             "mode": "absolute",
1830             "steps": [
1831               {
1832                 "color": "green",
1833                 "value": null
1834               }
1835             ]
1836           },
1837           "unit": "none"
1838         },
1839         "overrides": []
1840       },
1841       "gridPos": {
1842         "h": 4,
1843         "w": 4,
1844         "x": 16,
1845         "y": 23
1846       },
1847       "id": 38,
1848       "links": [],
1849       "maxDataPoints": 100,
1850       "options": {
1851         "colorMode": "value",
1852         "graphMode": "area",
1853         "justifyMode": "auto",
1854         "orientation": "horizontal",
1855         "reduceOptions": {
1856           "calcs": [
1857             "lastNotNull"
1858           ],
1859           "fields": "",
1860           "values": false
1861         },
1862         "text": {},
1863         "textMode": "auto"
1864       },
1865       "pluginVersion": "10.0.3",
1866       "targets": [
1867         {
1868           "datasource": {
1869             "type": "prometheus",
1870             "uid": "PBFA97CFB590B2093"
1871           },
1872           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1873           "interval": "",
1874           "legendFormat": "",
1875           "refId": "A"
1876         }
1877       ],
1878       "title": "Offset Commit Request Per Sec",
1879       "type": "stat"
1880     },
1881     {
1882       "datasource": {
1883         "type": "prometheus",
1884         "uid": "PBFA97CFB590B2093"
1885       },
1886       "description": "Fetch request rate.",
1887       "fieldConfig": {
1888         "defaults": {
1889           "mappings": [
1890             {
1891               "options": {
1892                 "match": "null",
1893                 "result": {
1894                   "text": "N/A"
1895                 }
1896               },
1897               "type": "special"
1898             }
1899           ],
1900           "thresholds": {
1901             "mode": "absolute",
1902             "steps": [
1903               {
1904                 "color": "green",
1905                 "value": null
1906               }
1907             ]
1908           },
1909           "unit": "none"
1910         },
1911         "overrides": []
1912       },
1913       "gridPos": {
1914         "h": 4,
1915         "w": 4,
1916         "x": 20,
1917         "y": 23
1918       },
1919       "id": 94,
1920       "links": [],
1921       "maxDataPoints": 100,
1922       "options": {
1923         "colorMode": "value",
1924         "graphMode": "area",
1925         "justifyMode": "auto",
1926         "orientation": "horizontal",
1927         "reduceOptions": {
1928           "calcs": [
1929             "lastNotNull"
1930           ],
1931           "fields": "",
1932           "values": false
1933         },
1934         "text": {},
1935         "textMode": "auto"
1936       },
1937       "pluginVersion": "10.0.3",
1938       "targets": [
1939         {
1940           "datasource": {
1941             "type": "prometheus",
1942             "uid": "PBFA97CFB590B2093"
1943           },
1944           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1945           "interval": "",
1946           "legendFormat": "",
1947           "refId": "A"
1948         }
1949       ],
1950       "title": "Broker Fetch Request Per Sec",
1951       "type": "stat"
1952     },
1953     {
1954       "collapsed": false,
1955       "datasource": {
1956         "type": "datasource",
1957         "uid": "grafana"
1958       },
1959       "gridPos": {
1960         "h": 1,
1961         "w": 24,
1962         "x": 0,
1963         "y": 27
1964       },
1965       "id": 29,
1966       "panels": [],
1967       "targets": [
1968         {
1969           "datasource": {
1970             "type": "datasource",
1971             "uid": "grafana"
1972           },
1973           "refId": "A"
1974         }
1975       ],
1976       "title": "Throughput In/Out",
1977       "type": "row"
1978     },
1979     {
1980       "datasource": {
1981         "type": "prometheus",
1982         "uid": "PBFA97CFB590B2093"
1983       },
1984       "fieldConfig": {
1985         "defaults": {
1986           "color": {
1987             "mode": "palette-classic"
1988           },
1989           "custom": {
1990             "axisCenteredZero": false,
1991             "axisColorMode": "text",
1992             "axisLabel": "Messages/s",
1993             "axisPlacement": "auto",
1994             "barAlignment": 0,
1995             "drawStyle": "line",
1996             "fillOpacity": 10,
1997             "gradientMode": "none",
1998             "hideFrom": {
1999               "legend": false,
2000               "tooltip": false,
2001               "viz": false
2002             },
2003             "lineInterpolation": "linear",
2004             "lineWidth": 2,
2005             "pointSize": 5,
2006             "scaleDistribution": {
2007               "type": "linear"
2008             },
2009             "showPoints": "never",
2010             "spanNulls": false,
2011             "stacking": {
2012               "group": "A",
2013               "mode": "none"
2014             },
2015             "thresholdsStyle": {
2016               "mode": "off"
2017             }
2018           },
2019           "links": [],
2020           "mappings": [],
2021           "min": 0,
2022           "thresholds": {
2023             "mode": "absolute",
2024             "steps": [
2025               {
2026                 "color": "green",
2027                 "value": null
2028               }
2029             ]
2030           },
2031           "unit": "cps"
2032         },
2033         "overrides": []
2034       },
2035       "gridPos": {
2036         "h": 8,
2037         "w": 8,
2038         "x": 0,
2039         "y": 28
2040       },
2041       "id": 4,
2042       "links": [],
2043       "options": {
2044         "legend": {
2045           "calcs": [
2046             "mean",
2047             "lastNotNull",
2048             "max"
2049           ],
2050           "displayMode": "table",
2051           "placement": "bottom",
2052           "showLegend": true
2053         },
2054         "tooltip": {
2055           "mode": "single",
2056           "sort": "none"
2057         }
2058       },
2059       "pluginVersion": "8.1.3",
2060       "targets": [
2061         {
2062           "datasource": {
2063             "type": "prometheus",
2064             "uid": "PBFA97CFB590B2093"
2065           },
2066           "exemplar": true,
2067           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2068           "interval": "",
2069           "intervalFactor": 2,
2070           "legendFormat": "bytes/sec",
2071           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2072           "refId": "A",
2073           "step": 4
2074         }
2075       ],
2076       "title": "Messages In",
2077       "type": "timeseries"
2078     },
2079     {
2080       "datasource": {
2081         "type": "prometheus",
2082         "uid": "PBFA97CFB590B2093"
2083       },
2084       "fieldConfig": {
2085         "defaults": {
2086           "color": {
2087             "mode": "palette-classic"
2088           },
2089           "custom": {
2090             "axisCenteredZero": false,
2091             "axisColorMode": "text",
2092             "axisLabel": "Bytes/s",
2093             "axisPlacement": "auto",
2094             "barAlignment": 0,
2095             "drawStyle": "line",
2096             "fillOpacity": 10,
2097             "gradientMode": "none",
2098             "hideFrom": {
2099               "legend": false,
2100               "tooltip": false,
2101               "viz": false
2102             },
2103             "lineInterpolation": "linear",
2104             "lineWidth": 2,
2105             "pointSize": 5,
2106             "scaleDistribution": {
2107               "type": "linear"
2108             },
2109             "showPoints": "never",
2110             "spanNulls": false,
2111             "stacking": {
2112               "group": "A",
2113               "mode": "none"
2114             },
2115             "thresholdsStyle": {
2116               "mode": "off"
2117             }
2118           },
2119           "links": [],
2120           "mappings": [],
2121           "min": 0,
2122           "thresholds": {
2123             "mode": "absolute",
2124             "steps": [
2125               {
2126                 "color": "green",
2127                 "value": null
2128               },
2129               {
2130                 "color": "red",
2131                 "value": 80
2132               }
2133             ]
2134           },
2135           "unit": "Bps"
2136         },
2137         "overrides": []
2138       },
2139       "gridPos": {
2140         "h": 8,
2141         "w": 8,
2142         "x": 8,
2143         "y": 28
2144       },
2145       "id": 5,
2146       "links": [],
2147       "options": {
2148         "legend": {
2149           "calcs": [
2150             "mean",
2151             "lastNotNull",
2152             "max"
2153           ],
2154           "displayMode": "table",
2155           "placement": "bottom",
2156           "showLegend": true
2157         },
2158         "tooltip": {
2159           "mode": "single",
2160           "sort": "none"
2161         }
2162       },
2163       "pluginVersion": "8.1.3",
2164       "targets": [
2165         {
2166           "datasource": {
2167             "type": "prometheus",
2168             "uid": "PBFA97CFB590B2093"
2169           },
2170           "exemplar": true,
2171           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2172           "format": "time_series",
2173           "interval": "",
2174           "intervalFactor": 2,
2175           "legendFormat": "bytes/sec",
2176           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2177           "refId": "A",
2178           "step": 4
2179         }
2180       ],
2181       "title": "Bytes In",
2182       "type": "timeseries"
2183     },
2184     {
2185       "datasource": {
2186         "type": "prometheus",
2187         "uid": "PBFA97CFB590B2093"
2188       },
2189       "fieldConfig": {
2190         "defaults": {
2191           "color": {
2192             "mode": "palette-classic"
2193           },
2194           "custom": {
2195             "axisCenteredZero": false,
2196             "axisColorMode": "text",
2197             "axisLabel": "Bytes/s",
2198             "axisPlacement": "auto",
2199             "barAlignment": 0,
2200             "drawStyle": "line",
2201             "fillOpacity": 10,
2202             "gradientMode": "none",
2203             "hideFrom": {
2204               "legend": false,
2205               "tooltip": false,
2206               "viz": false
2207             },
2208             "lineInterpolation": "linear",
2209             "lineWidth": 2,
2210             "pointSize": 5,
2211             "scaleDistribution": {
2212               "type": "linear"
2213             },
2214             "showPoints": "never",
2215             "spanNulls": false,
2216             "stacking": {
2217               "group": "A",
2218               "mode": "none"
2219             },
2220             "thresholdsStyle": {
2221               "mode": "off"
2222             }
2223           },
2224           "links": [],
2225           "mappings": [],
2226           "min": 0,
2227           "thresholds": {
2228             "mode": "absolute",
2229             "steps": [
2230               {
2231                 "color": "green",
2232                 "value": null
2233               },
2234               {
2235                 "color": "red",
2236                 "value": 80
2237               }
2238             ]
2239           },
2240           "unit": "Bps"
2241         },
2242         "overrides": []
2243       },
2244       "gridPos": {
2245         "h": 8,
2246         "w": 8,
2247         "x": 16,
2248         "y": 28
2249       },
2250       "id": 6,
2251       "links": [],
2252       "options": {
2253         "legend": {
2254           "calcs": [
2255             "mean",
2256             "lastNotNull",
2257             "max"
2258           ],
2259           "displayMode": "table",
2260           "placement": "bottom",
2261           "showLegend": true
2262         },
2263         "tooltip": {
2264           "mode": "single",
2265           "sort": "none"
2266         }
2267       },
2268       "pluginVersion": "8.1.3",
2269       "targets": [
2270         {
2271           "datasource": {
2272             "type": "prometheus",
2273             "uid": "PBFA97CFB590B2093"
2274           },
2275           "exemplar": true,
2276           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2277           "interval": "",
2278           "intervalFactor": 2,
2279           "legendFormat": "bytes/sec",
2280           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2281           "refId": "A",
2282           "step": 4
2283         }
2284       ],
2285       "title": "Bytes Out",
2286       "type": "timeseries"
2287     },
2288     {
2289       "datasource": {
2290         "type": "prometheus",
2291         "uid": "PBFA97CFB590B2093"
2292       },
2293       "fieldConfig": {
2294         "defaults": {
2295           "color": {
2296             "mode": "palette-classic"
2297           },
2298           "custom": {
2299             "axisCenteredZero": false,
2300             "axisColorMode": "text",
2301             "axisLabel": "Messages/s",
2302             "axisPlacement": "auto",
2303             "barAlignment": 0,
2304             "drawStyle": "line",
2305             "fillOpacity": 10,
2306             "gradientMode": "none",
2307             "hideFrom": {
2308               "legend": false,
2309               "tooltip": false,
2310               "viz": false
2311             },
2312             "lineInterpolation": "linear",
2313             "lineWidth": 2,
2314             "pointSize": 5,
2315             "scaleDistribution": {
2316               "type": "linear"
2317             },
2318             "showPoints": "never",
2319             "spanNulls": false,
2320             "stacking": {
2321               "group": "A",
2322               "mode": "none"
2323             },
2324             "thresholdsStyle": {
2325               "mode": "off"
2326             }
2327           },
2328           "links": [],
2329           "mappings": [],
2330           "thresholds": {
2331             "mode": "absolute",
2332             "steps": [
2333               {
2334                 "color": "green",
2335                 "value": null
2336               },
2337               {
2338                 "color": "red",
2339                 "value": 80
2340               }
2341             ]
2342           },
2343           "unit": "cps"
2344         },
2345         "overrides": []
2346       },
2347       "gridPos": {
2348         "h": 8,
2349         "w": 8,
2350         "x": 0,
2351         "y": 36
2352       },
2353       "id": 10,
2354       "links": [],
2355       "options": {
2356         "legend": {
2357           "calcs": [
2358             "mean",
2359             "lastNotNull",
2360             "max"
2361           ],
2362           "displayMode": "table",
2363           "placement": "bottom",
2364           "showLegend": true
2365         },
2366         "tooltip": {
2367           "mode": "single",
2368           "sort": "none"
2369         }
2370       },
2371       "pluginVersion": "8.1.3",
2372       "targets": [
2373         {
2374           "datasource": {
2375             "type": "prometheus",
2376             "uid": "PBFA97CFB590B2093"
2377           },
2378           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2379           "format": "time_series",
2380           "interval": "",
2381           "intervalFactor": 2,
2382           "legendFormat": "{{instance}}",
2383           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2384           "refId": "A",
2385           "step": 4
2386         }
2387       ],
2388       "title": "Messages In Per Broker",
2389       "type": "timeseries"
2390     },
2391     {
2392       "datasource": {
2393         "type": "prometheus",
2394         "uid": "PBFA97CFB590B2093"
2395       },
2396       "fieldConfig": {
2397         "defaults": {
2398           "color": {
2399             "mode": "palette-classic"
2400           },
2401           "custom": {
2402             "axisCenteredZero": false,
2403             "axisColorMode": "text",
2404             "axisLabel": "Bytes/s",
2405             "axisPlacement": "auto",
2406             "barAlignment": 0,
2407             "drawStyle": "line",
2408             "fillOpacity": 10,
2409             "gradientMode": "none",
2410             "hideFrom": {
2411               "legend": false,
2412               "tooltip": false,
2413               "viz": false
2414             },
2415             "lineInterpolation": "linear",
2416             "lineWidth": 2,
2417             "pointSize": 5,
2418             "scaleDistribution": {
2419               "type": "linear"
2420             },
2421             "showPoints": "never",
2422             "spanNulls": false,
2423             "stacking": {
2424               "group": "A",
2425               "mode": "none"
2426             },
2427             "thresholdsStyle": {
2428               "mode": "off"
2429             }
2430           },
2431           "links": [],
2432           "mappings": [],
2433           "thresholds": {
2434             "mode": "absolute",
2435             "steps": [
2436               {
2437                 "color": "green",
2438                 "value": null
2439               },
2440               {
2441                 "color": "red",
2442                 "value": 80
2443               }
2444             ]
2445           },
2446           "unit": "Bps"
2447         },
2448         "overrides": []
2449       },
2450       "gridPos": {
2451         "h": 8,
2452         "w": 8,
2453         "x": 8,
2454         "y": 36
2455       },
2456       "id": 7,
2457       "links": [],
2458       "options": {
2459         "legend": {
2460           "calcs": [
2461             "mean",
2462             "lastNotNull",
2463             "max"
2464           ],
2465           "displayMode": "table",
2466           "placement": "bottom",
2467           "showLegend": true
2468         },
2469         "tooltip": {
2470           "mode": "single",
2471           "sort": "none"
2472         }
2473       },
2474       "pluginVersion": "8.1.3",
2475       "targets": [
2476         {
2477           "datasource": {
2478             "type": "prometheus",
2479             "uid": "PBFA97CFB590B2093"
2480           },
2481           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2482           "format": "time_series",
2483           "interval": "",
2484           "intervalFactor": 2,
2485           "legendFormat": "{{instance}}",
2486           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2487           "refId": "A",
2488           "step": 4
2489         }
2490       ],
2491       "title": "Bytes In Per Broker",
2492       "type": "timeseries"
2493     },
2494     {
2495       "datasource": {
2496         "type": "prometheus",
2497         "uid": "PBFA97CFB590B2093"
2498       },
2499       "fieldConfig": {
2500         "defaults": {
2501           "color": {
2502             "mode": "palette-classic"
2503           },
2504           "custom": {
2505             "axisCenteredZero": false,
2506             "axisColorMode": "text",
2507             "axisLabel": "",
2508             "axisPlacement": "auto",
2509             "barAlignment": 0,
2510             "drawStyle": "line",
2511             "fillOpacity": 10,
2512             "gradientMode": "none",
2513             "hideFrom": {
2514               "legend": false,
2515               "tooltip": false,
2516               "viz": false
2517             },
2518             "lineInterpolation": "linear",
2519             "lineWidth": 1,
2520             "pointSize": 5,
2521             "scaleDistribution": {
2522               "type": "linear"
2523             },
2524             "showPoints": "never",
2525             "spanNulls": true,
2526             "stacking": {
2527               "group": "A",
2528               "mode": "none"
2529             },
2530             "thresholdsStyle": {
2531               "mode": "off"
2532             }
2533           },
2534           "links": [],
2535           "mappings": [],
2536           "min": 0,
2537           "thresholds": {
2538             "mode": "absolute",
2539             "steps": [
2540               {
2541                 "color": "green",
2542                 "value": null
2543               },
2544               {
2545                 "color": "red",
2546                 "value": 80
2547               }
2548             ]
2549           },
2550           "unit": "binBps"
2551         },
2552         "overrides": []
2553       },
2554       "gridPos": {
2555         "h": 8,
2556         "w": 8,
2557         "x": 16,
2558         "y": 36
2559       },
2560       "id": 9,
2561       "links": [],
2562       "options": {
2563         "legend": {
2564           "calcs": [
2565             "mean",
2566             "lastNotNull",
2567             "max"
2568           ],
2569           "displayMode": "table",
2570           "placement": "bottom",
2571           "showLegend": true
2572         },
2573         "tooltip": {
2574           "mode": "single",
2575           "sort": "none"
2576         }
2577       },
2578       "pluginVersion": "8.1.3",
2579       "targets": [
2580         {
2581           "datasource": {
2582             "type": "prometheus",
2583             "uid": "PBFA97CFB590B2093"
2584           },
2585           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2586           "format": "time_series",
2587           "interval": "",
2588           "intervalFactor": 1,
2589           "legendFormat": "{{instance}}",
2590           "refId": "A"
2591         }
2592       ],
2593       "title": "Bytes Out Per Broker",
2594       "type": "timeseries"
2595     },
2596     {
2597       "collapsed": false,
2598       "datasource": {
2599         "type": "datasource",
2600         "uid": "grafana"
2601       },
2602       "gridPos": {
2603         "h": 1,
2604         "w": 24,
2605         "x": 0,
2606         "y": 44
2607       },
2608       "id": 44,
2609       "panels": [],
2610       "targets": [
2611         {
2612           "datasource": {
2613             "type": "datasource",
2614             "uid": "grafana"
2615           },
2616           "refId": "A"
2617         }
2618       ],
2619       "title": "Thread utilization",
2620       "type": "row"
2621     },
2622     {
2623       "datasource": {
2624         "type": "prometheus",
2625         "uid": "PBFA97CFB590B2093"
2626       },
2627       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2628       "fieldConfig": {
2629         "defaults": {
2630           "color": {
2631             "mode": "palette-classic"
2632           },
2633           "custom": {
2634             "axisCenteredZero": false,
2635             "axisColorMode": "text",
2636             "axisLabel": "",
2637             "axisPlacement": "auto",
2638             "barAlignment": 0,
2639             "drawStyle": "line",
2640             "fillOpacity": 10,
2641             "gradientMode": "none",
2642             "hideFrom": {
2643               "legend": false,
2644               "tooltip": false,
2645               "viz": false
2646             },
2647             "lineInterpolation": "linear",
2648             "lineWidth": 1,
2649             "pointSize": 5,
2650             "scaleDistribution": {
2651               "type": "linear"
2652             },
2653             "showPoints": "never",
2654             "spanNulls": true,
2655             "stacking": {
2656               "group": "A",
2657               "mode": "none"
2658             },
2659             "thresholdsStyle": {
2660               "mode": "off"
2661             }
2662           },
2663           "links": [],
2664           "mappings": [],
2665           "thresholds": {
2666             "mode": "absolute",
2667             "steps": [
2668               {
2669                 "color": "green",
2670                 "value": null
2671               },
2672               {
2673                 "color": "red",
2674                 "value": 80
2675               }
2676             ]
2677           },
2678           "unit": "percentunit"
2679         },
2680         "overrides": []
2681       },
2682       "gridPos": {
2683         "h": 12,
2684         "w": 12,
2685         "x": 0,
2686         "y": 45
2687       },
2688       "id": 24,
2689       "options": {
2690         "legend": {
2691           "calcs": [
2692             "mean",
2693             "lastNotNull",
2694             "max"
2695           ],
2696           "displayMode": "table",
2697           "placement": "bottom",
2698           "showLegend": true
2699         },
2700         "tooltip": {
2701           "mode": "single",
2702           "sort": "none"
2703         }
2704       },
2705       "pluginVersion": "8.1.3",
2706       "targets": [
2707         {
2708           "datasource": {
2709             "type": "prometheus",
2710             "uid": "PBFA97CFB590B2093"
2711           },
2712           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2713           "interval": "",
2714           "legendFormat": "{{instance}}",
2715           "refId": "A"
2716         }
2717       ],
2718       "title": "Network Processor Avg Usage Percent",
2719       "type": "timeseries"
2720     },
2721     {
2722       "datasource": {
2723         "type": "prometheus",
2724         "uid": "PBFA97CFB590B2093"
2725       },
2726       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2727       "fieldConfig": {
2728         "defaults": {
2729           "color": {
2730             "mode": "palette-classic"
2731           },
2732           "custom": {
2733             "axisCenteredZero": false,
2734             "axisColorMode": "text",
2735             "axisLabel": "",
2736             "axisPlacement": "auto",
2737             "barAlignment": 0,
2738             "drawStyle": "line",
2739             "fillOpacity": 10,
2740             "gradientMode": "none",
2741             "hideFrom": {
2742               "legend": false,
2743               "tooltip": false,
2744               "viz": false
2745             },
2746             "lineInterpolation": "linear",
2747             "lineWidth": 1,
2748             "pointSize": 5,
2749             "scaleDistribution": {
2750               "type": "linear"
2751             },
2752             "showPoints": "never",
2753             "spanNulls": true,
2754             "stacking": {
2755               "group": "A",
2756               "mode": "none"
2757             },
2758             "thresholdsStyle": {
2759               "mode": "off"
2760             }
2761           },
2762           "links": [],
2763           "mappings": [],
2764           "min": 0,
2765           "thresholds": {
2766             "mode": "absolute",
2767             "steps": [
2768               {
2769                 "color": "green",
2770                 "value": null
2771               },
2772               {
2773                 "color": "red",
2774                 "value": 80
2775               }
2776             ]
2777           },
2778           "unit": "percentunit"
2779         },
2780         "overrides": []
2781       },
2782       "gridPos": {
2783         "h": 12,
2784         "w": 12,
2785         "x": 12,
2786         "y": 45
2787       },
2788       "id": 25,
2789       "options": {
2790         "legend": {
2791           "calcs": [
2792             "mean",
2793             "lastNotNull",
2794             "max"
2795           ],
2796           "displayMode": "table",
2797           "placement": "bottom",
2798           "showLegend": true
2799         },
2800         "tooltip": {
2801           "mode": "single",
2802           "sort": "none"
2803         }
2804       },
2805       "pluginVersion": "8.1.3",
2806       "targets": [
2807         {
2808           "datasource": {
2809             "type": "prometheus",
2810             "uid": "PBFA97CFB590B2093"
2811           },
2812           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2813           "interval": "",
2814           "legendFormat": "{{instance}}",
2815           "refId": "A"
2816         }
2817       ],
2818       "title": "Request Handler Avg Percent",
2819       "type": "timeseries"
2820     },
2821     {
2822       "collapsed": false,
2823       "datasource": {
2824         "type": "datasource",
2825         "uid": "grafana"
2826       },
2827       "gridPos": {
2828         "h": 1,
2829         "w": 24,
2830         "x": 0,
2831         "y": 57
2832       },
2833       "id": 82,
2834       "panels": [],
2835       "targets": [
2836         {
2837           "datasource": {
2838             "type": "datasource",
2839             "uid": "grafana"
2840           },
2841           "refId": "A"
2842         }
2843       ],
2844       "title": "Isr Shrinks / Expands",
2845       "type": "row"
2846     },
2847     {
2848       "datasource": {
2849         "type": "prometheus",
2850         "uid": "PBFA97CFB590B2093"
2851       },
2852       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2853       "fieldConfig": {
2854         "defaults": {
2855           "color": {
2856             "mode": "palette-classic"
2857           },
2858           "custom": {
2859             "axisCenteredZero": false,
2860             "axisColorMode": "text",
2861             "axisLabel": "",
2862             "axisPlacement": "auto",
2863             "barAlignment": 0,
2864             "drawStyle": "line",
2865             "fillOpacity": 10,
2866             "gradientMode": "none",
2867             "hideFrom": {
2868               "legend": false,
2869               "tooltip": false,
2870               "viz": false
2871             },
2872             "lineInterpolation": "linear",
2873             "lineWidth": 1,
2874             "pointSize": 5,
2875             "scaleDistribution": {
2876               "type": "linear"
2877             },
2878             "showPoints": "never",
2879             "spanNulls": true,
2880             "stacking": {
2881               "group": "A",
2882               "mode": "none"
2883             },
2884             "thresholdsStyle": {
2885               "mode": "off"
2886             }
2887           },
2888           "links": [],
2889           "mappings": [],
2890           "min": 0,
2891           "thresholds": {
2892             "mode": "absolute",
2893             "steps": [
2894               {
2895                 "color": "green",
2896                 "value": null
2897               },
2898               {
2899                 "color": "red",
2900                 "value": 80
2901               }
2902             ]
2903           },
2904           "unit": "ops"
2905         },
2906         "overrides": [
2907           {
2908             "__systemRef": "hideSeriesFrom",
2909             "matcher": {
2910               "id": "byNames",
2911               "options": {
2912                 "mode": "exclude",
2913                 "names": [
2914                   "kafka-2:80"
2915                 ],
2916                 "prefix": "All except:",
2917                 "readOnly": true
2918               }
2919             },
2920             "properties": [
2921               {
2922                 "id": "custom.hideFrom",
2923                 "value": {
2924                   "legend": false,
2925                   "tooltip": false,
2926                   "viz": true
2927                 }
2928               }
2929             ]
2930           }
2931         ]
2932       },
2933       "gridPos": {
2934         "h": 9,
2935         "w": 12,
2936         "x": 0,
2937         "y": 58
2938       },
2939       "id": 80,
2940       "options": {
2941         "legend": {
2942           "calcs": [
2943             "mean",
2944             "lastNotNull",
2945             "max"
2946           ],
2947           "displayMode": "table",
2948           "placement": "bottom",
2949           "showLegend": true
2950         },
2951         "tooltip": {
2952           "mode": "single",
2953           "sort": "none"
2954         }
2955       },
2956       "pluginVersion": "8.1.3",
2957       "targets": [
2958         {
2959           "datasource": {
2960             "type": "prometheus",
2961             "uid": "PBFA97CFB590B2093"
2962           },
2963           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2964           "interval": "",
2965           "legendFormat": "{{instance}}",
2966           "refId": "A"
2967         }
2968       ],
2969       "title": "IsrShrinks per Sec",
2970       "type": "timeseries"
2971     },
2972     {
2973       "datasource": {
2974         "type": "prometheus",
2975         "uid": "PBFA97CFB590B2093"
2976       },
2977       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2978       "fieldConfig": {
2979         "defaults": {
2980           "color": {
2981             "mode": "palette-classic"
2982           },
2983           "custom": {
2984             "axisCenteredZero": false,
2985             "axisColorMode": "text",
2986             "axisLabel": "",
2987             "axisPlacement": "auto",
2988             "barAlignment": 0,
2989             "drawStyle": "line",
2990             "fillOpacity": 10,
2991             "gradientMode": "none",
2992             "hideFrom": {
2993               "legend": false,
2994               "tooltip": false,
2995               "viz": false
2996             },
2997             "lineInterpolation": "linear",
2998             "lineWidth": 1,
2999             "pointSize": 5,
3000             "scaleDistribution": {
3001               "type": "linear"
3002             },
3003             "showPoints": "never",
3004             "spanNulls": true,
3005             "stacking": {
3006               "group": "A",
3007               "mode": "none"
3008             },
3009             "thresholdsStyle": {
3010               "mode": "off"
3011             }
3012           },
3013           "links": [],
3014           "mappings": [],
3015           "min": 0,
3016           "thresholds": {
3017             "mode": "absolute",
3018             "steps": [
3019               {
3020                 "color": "green",
3021                 "value": null
3022               },
3023               {
3024                 "color": "red",
3025                 "value": 80
3026               }
3027             ]
3028           },
3029           "unit": "ops"
3030         },
3031         "overrides": [
3032           {
3033             "__systemRef": "hideSeriesFrom",
3034             "matcher": {
3035               "id": "byNames",
3036               "options": {
3037                 "mode": "exclude",
3038                 "names": [
3039                   "kafka-2:80"
3040                 ],
3041                 "prefix": "All except:",
3042                 "readOnly": true
3043               }
3044             },
3045             "properties": [
3046               {
3047                 "id": "custom.hideFrom",
3048                 "value": {
3049                   "legend": false,
3050                   "tooltip": false,
3051                   "viz": true
3052                 }
3053               }
3054             ]
3055           }
3056         ]
3057       },
3058       "gridPos": {
3059         "h": 9,
3060         "w": 12,
3061         "x": 12,
3062         "y": 58
3063       },
3064       "id": 83,
3065       "options": {
3066         "legend": {
3067           "calcs": [
3068             "mean",
3069             "lastNotNull",
3070             "max"
3071           ],
3072           "displayMode": "table",
3073           "placement": "bottom",
3074           "showLegend": true
3075         },
3076         "tooltip": {
3077           "mode": "single",
3078           "sort": "none"
3079         }
3080       },
3081       "pluginVersion": "8.1.3",
3082       "targets": [
3083         {
3084           "datasource": {
3085             "type": "prometheus",
3086             "uid": "PBFA97CFB590B2093"
3087           },
3088           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3089           "hide": false,
3090           "interval": "",
3091           "legendFormat": "{{instance}}",
3092           "refId": "A"
3093         }
3094       ],
3095       "title": "IsrExpands per Sec",
3096       "type": "timeseries"
3097     },
3098     {
3099       "collapsed": true,
3100       "datasource": {
3101         "type": "datasource",
3102         "uid": "grafana"
3103       },
3104       "gridPos": {
3105         "h": 1,
3106         "w": 24,
3107         "x": 0,
3108         "y": 67
3109       },
3110       "id": 53,
3111       "panels": [
3112         {
3113           "datasource": {
3114             "type": "prometheus",
3115             "uid": "PBFA97CFB590B2093"
3116           },
3117           "fieldConfig": {
3118             "defaults": {
3119               "color": {
3120                 "mode": "palette-classic"
3121               },
3122               "custom": {
3123                 "axisLabel": "",
3124                 "axisPlacement": "auto",
3125                 "barAlignment": 0,
3126                 "drawStyle": "line",
3127                 "fillOpacity": 10,
3128                 "gradientMode": "none",
3129                 "hideFrom": {
3130                   "legend": false,
3131                   "tooltip": false,
3132                   "viz": false
3133                 },
3134                 "lineInterpolation": "linear",
3135                 "lineWidth": 1,
3136                 "pointSize": 5,
3137                 "scaleDistribution": {
3138                   "type": "linear"
3139                 },
3140                 "showPoints": "never",
3141                 "spanNulls": true,
3142                 "stacking": {
3143                   "group": "A",
3144                   "mode": "none"
3145                 },
3146                 "thresholdsStyle": {
3147                   "mode": "off"
3148                 }
3149               },
3150               "links": [],
3151               "mappings": [],
3152               "thresholds": {
3153                 "mode": "absolute",
3154                 "steps": [
3155                   {
3156                     "color": "green"
3157                   },
3158                   {
3159                     "color": "red",
3160                     "value": 80
3161                   }
3162                 ]
3163               },
3164               "unit": "bytes"
3165             },
3166             "overrides": []
3167           },
3168           "gridPos": {
3169             "h": 9,
3170             "w": 12,
3171             "x": 0,
3172             "y": 28
3173           },
3174           "id": 55,
3175           "options": {
3176             "legend": {
3177               "calcs": [
3178                 "mean",
3179                 "lastNotNull",
3180                 "max"
3181               ],
3182               "displayMode": "table",
3183               "placement": "bottom",
3184               "showLegend": true
3185             },
3186             "tooltip": {
3187               "mode": "single"
3188             }
3189           },
3190           "pluginVersion": "8.1.3",
3191           "targets": [
3192             {
3193               "datasource": {
3194                 "type": "prometheus",
3195                 "uid": "PBFA97CFB590B2093"
3196               },
3197               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3198               "legendFormat": "{{topic}}",
3199               "refId": "A"
3200             }
3201           ],
3202           "title": "Log size per Topic",
3203           "type": "timeseries"
3204         },
3205         {
3206           "datasource": {
3207             "type": "prometheus",
3208             "uid": "PBFA97CFB590B2093"
3209           },
3210           "fieldConfig": {
3211             "defaults": {
3212               "color": {
3213                 "mode": "palette-classic"
3214               },
3215               "custom": {
3216                 "axisLabel": "",
3217                 "axisPlacement": "auto",
3218                 "barAlignment": 0,
3219                 "drawStyle": "line",
3220                 "fillOpacity": 10,
3221                 "gradientMode": "none",
3222                 "hideFrom": {
3223                   "legend": false,
3224                   "tooltip": false,
3225                   "viz": false
3226                 },
3227                 "lineInterpolation": "linear",
3228                 "lineWidth": 1,
3229                 "pointSize": 5,
3230                 "scaleDistribution": {
3231                   "type": "linear"
3232                 },
3233                 "showPoints": "never",
3234                 "spanNulls": true,
3235                 "stacking": {
3236                   "group": "A",
3237                   "mode": "none"
3238                 },
3239                 "thresholdsStyle": {
3240                   "mode": "off"
3241                 }
3242               },
3243               "links": [],
3244               "mappings": [],
3245               "thresholds": {
3246                 "mode": "absolute",
3247                 "steps": [
3248                   {
3249                     "color": "green"
3250                   },
3251                   {
3252                     "color": "red",
3253                     "value": 80
3254                   }
3255                 ]
3256               },
3257               "unit": "bytes"
3258             },
3259             "overrides": []
3260           },
3261           "gridPos": {
3262             "h": 9,
3263             "w": 12,
3264             "x": 12,
3265             "y": 28
3266           },
3267           "id": 56,
3268           "options": {
3269             "legend": {
3270               "calcs": [
3271                 "mean",
3272                 "lastNotNull",
3273                 "max"
3274               ],
3275               "displayMode": "table",
3276               "placement": "bottom",
3277               "showLegend": true
3278             },
3279             "tooltip": {
3280               "mode": "single"
3281             }
3282           },
3283           "pluginVersion": "8.1.3",
3284           "targets": [
3285             {
3286               "datasource": {
3287                 "type": "prometheus",
3288                 "uid": "PBFA97CFB590B2093"
3289               },
3290               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3291               "legendFormat": "{{instance}}",
3292               "refId": "A"
3293             }
3294           ],
3295           "title": "Log size per Broker",
3296           "type": "timeseries"
3297         }
3298       ],
3299       "targets": [
3300         {
3301           "datasource": {
3302             "type": "datasource",
3303             "uid": "grafana"
3304           },
3305           "refId": "A"
3306         }
3307       ],
3308       "title": "Logs size",
3309       "type": "row"
3310     },
3311     {
3312       "collapsed": true,
3313       "datasource": {
3314         "type": "datasource",
3315         "uid": "grafana"
3316       },
3317       "gridPos": {
3318         "h": 1,
3319         "w": 24,
3320         "x": 0,
3321         "y": 68
3322       },
3323       "id": 58,
3324       "panels": [
3325         {
3326           "datasource": {
3327             "type": "prometheus",
3328             "uid": "PBFA97CFB590B2093"
3329           },
3330           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3331           "fieldConfig": {
3332             "defaults": {
3333               "color": {
3334                 "mode": "palette-classic"
3335               },
3336               "custom": {
3337                 "axisLabel": "",
3338                 "axisPlacement": "auto",
3339                 "barAlignment": 0,
3340                 "drawStyle": "line",
3341                 "fillOpacity": 10,
3342                 "gradientMode": "none",
3343                 "hideFrom": {
3344                   "legend": false,
3345                   "tooltip": false,
3346                   "viz": false
3347                 },
3348                 "lineInterpolation": "linear",
3349                 "lineWidth": 1,
3350                 "pointSize": 5,
3351                 "scaleDistribution": {
3352                   "type": "linear"
3353                 },
3354                 "showPoints": "never",
3355                 "spanNulls": true,
3356                 "stacking": {
3357                   "group": "A",
3358                   "mode": "none"
3359                 },
3360                 "thresholdsStyle": {
3361                   "mode": "off"
3362                 }
3363               },
3364               "links": [],
3365               "mappings": [],
3366               "thresholds": {
3367                 "mode": "absolute",
3368                 "steps": [
3369                   {
3370                     "color": "green"
3371                   },
3372                   {
3373                     "color": "red",
3374                     "value": 80
3375                   }
3376                 ]
3377               },
3378               "unit": "ms"
3379             },
3380             "overrides": []
3381           },
3382           "gridPos": {
3383             "h": 9,
3384             "w": 12,
3385             "x": 0,
3386             "y": 29
3387           },
3388           "id": 60,
3389           "options": {
3390             "legend": {
3391               "calcs": [
3392                 "mean",
3393                 "lastNotNull",
3394                 "max"
3395               ],
3396               "displayMode": "list",
3397               "placement": "bottom",
3398               "showLegend": true
3399             },
3400             "tooltip": {
3401               "mode": "single"
3402             }
3403           },
3404           "pluginVersion": "8.1.3",
3405           "targets": [
3406             {
3407               "datasource": {
3408                 "type": "prometheus",
3409                 "uid": "PBFA97CFB590B2093"
3410               },
3411               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3412               "hide": false,
3413               "legendFormat": "{{instance}} - {{quantile}}",
3414               "refId": "A"
3415             }
3416           ],
3417           "title": "Producer - RequestQueueTimeMs",
3418           "type": "timeseries"
3419         },
3420         {
3421           "datasource": {
3422             "type": "prometheus",
3423             "uid": "PBFA97CFB590B2093"
3424           },
3425           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3426           "fieldConfig": {
3427             "defaults": {
3428               "color": {
3429                 "mode": "palette-classic"
3430               },
3431               "custom": {
3432                 "axisLabel": "",
3433                 "axisPlacement": "auto",
3434                 "barAlignment": 0,
3435                 "drawStyle": "line",
3436                 "fillOpacity": 10,
3437                 "gradientMode": "none",
3438                 "hideFrom": {
3439                   "legend": false,
3440                   "tooltip": false,
3441                   "viz": false
3442                 },
3443                 "lineInterpolation": "linear",
3444                 "lineWidth": 1,
3445                 "pointSize": 5,
3446                 "scaleDistribution": {
3447                   "type": "linear"
3448                 },
3449                 "showPoints": "never",
3450                 "spanNulls": true,
3451                 "stacking": {
3452                   "group": "A",
3453                   "mode": "none"
3454                 },
3455                 "thresholdsStyle": {
3456                   "mode": "off"
3457                 }
3458               },
3459               "links": [],
3460               "mappings": [],
3461               "thresholds": {
3462                 "mode": "absolute",
3463                 "steps": [
3464                   {
3465                     "color": "green"
3466                   },
3467                   {
3468                     "color": "red",
3469                     "value": 80
3470                   }
3471                 ]
3472               },
3473               "unit": "ms"
3474             },
3475             "overrides": []
3476           },
3477           "gridPos": {
3478             "h": 9,
3479             "w": 12,
3480             "x": 12,
3481             "y": 29
3482           },
3483           "id": 61,
3484           "options": {
3485             "legend": {
3486               "calcs": [
3487                 "mean",
3488                 "lastNotNull",
3489                 "max"
3490               ],
3491               "displayMode": "list",
3492               "placement": "bottom",
3493               "showLegend": true
3494             },
3495             "tooltip": {
3496               "mode": "single"
3497             }
3498           },
3499           "pluginVersion": "8.1.3",
3500           "targets": [
3501             {
3502               "datasource": {
3503                 "type": "prometheus",
3504                 "uid": "PBFA97CFB590B2093"
3505               },
3506               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3507               "hide": false,
3508               "legendFormat": "{{instance}} - {{quantile}}",
3509               "refId": "A"
3510             }
3511           ],
3512           "title": "Producer - LocalTimeMs",
3513           "type": "timeseries"
3514         },
3515         {
3516           "datasource": {
3517             "type": "prometheus",
3518             "uid": "PBFA97CFB590B2093"
3519           },
3520           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3521           "fieldConfig": {
3522             "defaults": {
3523               "color": {
3524                 "mode": "palette-classic"
3525               },
3526               "custom": {
3527                 "axisLabel": "",
3528                 "axisPlacement": "auto",
3529                 "barAlignment": 0,
3530                 "drawStyle": "line",
3531                 "fillOpacity": 10,
3532                 "gradientMode": "none",
3533                 "hideFrom": {
3534                   "legend": false,
3535                   "tooltip": false,
3536                   "viz": false
3537                 },
3538                 "lineInterpolation": "linear",
3539                 "lineWidth": 1,
3540                 "pointSize": 5,
3541                 "scaleDistribution": {
3542                   "type": "linear"
3543                 },
3544                 "showPoints": "never",
3545                 "spanNulls": true,
3546                 "stacking": {
3547                   "group": "A",
3548                   "mode": "none"
3549                 },
3550                 "thresholdsStyle": {
3551                   "mode": "off"
3552                 }
3553               },
3554               "links": [],
3555               "mappings": [],
3556               "thresholds": {
3557                 "mode": "absolute",
3558                 "steps": [
3559                   {
3560                     "color": "green"
3561                   },
3562                   {
3563                     "color": "red",
3564                     "value": 80
3565                   }
3566                 ]
3567               },
3568               "unit": "ms"
3569             },
3570             "overrides": []
3571           },
3572           "gridPos": {
3573             "h": 10,
3574             "w": 8,
3575             "x": 0,
3576             "y": 38
3577           },
3578           "id": 62,
3579           "options": {
3580             "legend": {
3581               "calcs": [
3582                 "mean",
3583                 "lastNotNull",
3584                 "max"
3585               ],
3586               "displayMode": "table",
3587               "placement": "bottom",
3588               "showLegend": true
3589             },
3590             "tooltip": {
3591               "mode": "single"
3592             }
3593           },
3594           "pluginVersion": "8.1.3",
3595           "targets": [
3596             {
3597               "datasource": {
3598                 "type": "prometheus",
3599                 "uid": "PBFA97CFB590B2093"
3600               },
3601               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3602               "hide": false,
3603               "legendFormat": "{{instance}} - {{quantile}}",
3604               "refId": "A"
3605             }
3606           ],
3607           "title": "Producer - RemoteTimeMs",
3608           "type": "timeseries"
3609         },
3610         {
3611           "datasource": {
3612             "type": "prometheus",
3613             "uid": "PBFA97CFB590B2093"
3614           },
3615           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3616           "fieldConfig": {
3617             "defaults": {
3618               "color": {
3619                 "mode": "palette-classic"
3620               },
3621               "custom": {
3622                 "axisLabel": "",
3623                 "axisPlacement": "auto",
3624                 "barAlignment": 0,
3625                 "drawStyle": "line",
3626                 "fillOpacity": 10,
3627                 "gradientMode": "none",
3628                 "hideFrom": {
3629                   "legend": false,
3630                   "tooltip": false,
3631                   "viz": false
3632                 },
3633                 "lineInterpolation": "linear",
3634                 "lineWidth": 1,
3635                 "pointSize": 5,
3636                 "scaleDistribution": {
3637                   "type": "linear"
3638                 },
3639                 "showPoints": "never",
3640                 "spanNulls": true,
3641                 "stacking": {
3642                   "group": "A",
3643                   "mode": "none"
3644                 },
3645                 "thresholdsStyle": {
3646                   "mode": "off"
3647                 }
3648               },
3649               "links": [],
3650               "mappings": [],
3651               "thresholds": {
3652                 "mode": "absolute",
3653                 "steps": [
3654                   {
3655                     "color": "green"
3656                   },
3657                   {
3658                     "color": "red",
3659                     "value": 80
3660                   }
3661                 ]
3662               },
3663               "unit": "ms"
3664             },
3665             "overrides": []
3666           },
3667           "gridPos": {
3668             "h": 10,
3669             "w": 8,
3670             "x": 8,
3671             "y": 38
3672           },
3673           "id": 63,
3674           "options": {
3675             "legend": {
3676               "calcs": [
3677                 "mean",
3678                 "lastNotNull",
3679                 "max"
3680               ],
3681               "displayMode": "table",
3682               "placement": "bottom",
3683               "showLegend": true
3684             },
3685             "tooltip": {
3686               "mode": "single"
3687             }
3688           },
3689           "pluginVersion": "8.1.3",
3690           "targets": [
3691             {
3692               "datasource": {
3693                 "type": "prometheus",
3694                 "uid": "PBFA97CFB590B2093"
3695               },
3696               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3697               "hide": false,
3698               "legendFormat": "{{instance}} - {{quantile}}",
3699               "refId": "A"
3700             }
3701           ],
3702           "title": "Producer - ResponseQueueTimeMs",
3703           "type": "timeseries"
3704         },
3705         {
3706           "datasource": {
3707             "type": "prometheus",
3708             "uid": "PBFA97CFB590B2093"
3709           },
3710           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3711           "fieldConfig": {
3712             "defaults": {
3713               "color": {
3714                 "mode": "palette-classic"
3715               },
3716               "custom": {
3717                 "axisLabel": "",
3718                 "axisPlacement": "auto",
3719                 "barAlignment": 0,
3720                 "drawStyle": "line",
3721                 "fillOpacity": 10,
3722                 "gradientMode": "none",
3723                 "hideFrom": {
3724                   "legend": false,
3725                   "tooltip": false,
3726                   "viz": false
3727                 },
3728                 "lineInterpolation": "linear",
3729                 "lineWidth": 1,
3730                 "pointSize": 5,
3731                 "scaleDistribution": {
3732                   "type": "linear"
3733                 },
3734                 "showPoints": "never",
3735                 "spanNulls": true,
3736                 "stacking": {
3737                   "group": "A",
3738                   "mode": "none"
3739                 },
3740                 "thresholdsStyle": {
3741                   "mode": "off"
3742                 }
3743               },
3744               "links": [],
3745               "mappings": [],
3746               "min": 0,
3747               "thresholds": {
3748                 "mode": "absolute",
3749                 "steps": [
3750                   {
3751                     "color": "green"
3752                   },
3753                   {
3754                     "color": "red",
3755                     "value": 80
3756                   }
3757                 ]
3758               },
3759               "unit": "ms"
3760             },
3761             "overrides": []
3762           },
3763           "gridPos": {
3764             "h": 10,
3765             "w": 8,
3766             "x": 16,
3767             "y": 38
3768           },
3769           "id": 64,
3770           "options": {
3771             "legend": {
3772               "calcs": [
3773                 "mean",
3774                 "lastNotNull",
3775                 "max"
3776               ],
3777               "displayMode": "table",
3778               "placement": "bottom",
3779               "showLegend": true
3780             },
3781             "tooltip": {
3782               "mode": "single"
3783             }
3784           },
3785           "pluginVersion": "8.1.3",
3786           "targets": [
3787             {
3788               "datasource": {
3789                 "type": "prometheus",
3790                 "uid": "PBFA97CFB590B2093"
3791               },
3792               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3793               "hide": false,
3794               "legendFormat": "{{instance}} - {{quantile}}",
3795               "refId": "A"
3796             }
3797           ],
3798           "title": "Producer - ResponseSendTimeMs",
3799           "type": "timeseries"
3800         }
3801       ],
3802       "targets": [
3803         {
3804           "datasource": {
3805             "type": "datasource",
3806             "uid": "grafana"
3807           },
3808           "refId": "A"
3809         }
3810       ],
3811       "title": "Producer Performance",
3812       "type": "row"
3813     },
3814     {
3815       "collapsed": true,
3816       "datasource": {
3817         "type": "datasource",
3818         "uid": "grafana"
3819       },
3820       "gridPos": {
3821         "h": 1,
3822         "w": 24,
3823         "x": 0,
3824         "y": 69
3825       },
3826       "id": 68,
3827       "panels": [
3828         {
3829           "datasource": {
3830             "type": "prometheus",
3831             "uid": "PBFA97CFB590B2093"
3832           },
3833           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3834           "fieldConfig": {
3835             "defaults": {
3836               "color": {
3837                 "mode": "palette-classic"
3838               },
3839               "custom": {
3840                 "axisLabel": "",
3841                 "axisPlacement": "auto",
3842                 "barAlignment": 0,
3843                 "drawStyle": "line",
3844                 "fillOpacity": 10,
3845                 "gradientMode": "none",
3846                 "hideFrom": {
3847                   "legend": false,
3848                   "tooltip": false,
3849                   "viz": false
3850                 },
3851                 "lineInterpolation": "linear",
3852                 "lineWidth": 1,
3853                 "pointSize": 5,
3854                 "scaleDistribution": {
3855                   "type": "linear"
3856                 },
3857                 "showPoints": "never",
3858                 "spanNulls": true,
3859                 "stacking": {
3860                   "group": "A",
3861                   "mode": "none"
3862                 },
3863                 "thresholdsStyle": {
3864                   "mode": "off"
3865                 }
3866               },
3867               "links": [],
3868               "mappings": [],
3869               "min": 0,
3870               "thresholds": {
3871                 "mode": "absolute",
3872                 "steps": [
3873                   {
3874                     "color": "green"
3875                   },
3876                   {
3877                     "color": "red",
3878                     "value": 80
3879                   }
3880                 ]
3881               },
3882               "unit": "ms"
3883             },
3884             "overrides": []
3885           },
3886           "gridPos": {
3887             "h": 9,
3888             "w": 12,
3889             "x": 0,
3890             "y": 30
3891           },
3892           "id": 69,
3893           "options": {
3894             "legend": {
3895               "calcs": [
3896                 "mean",
3897                 "lastNotNull",
3898                 "max"
3899               ],
3900               "displayMode": "list",
3901               "placement": "bottom",
3902               "showLegend": true
3903             },
3904             "tooltip": {
3905               "mode": "single"
3906             }
3907           },
3908           "pluginVersion": "8.1.3",
3909           "targets": [
3910             {
3911               "datasource": {
3912                 "type": "prometheus",
3913                 "uid": "PBFA97CFB590B2093"
3914               },
3915               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3916               "hide": false,
3917               "legendFormat": "{{instance}} - {{quantile}}",
3918               "refId": "A"
3919             }
3920           ],
3921           "title": "Consumer - RequestQueueTimeMs",
3922           "type": "timeseries"
3923         },
3924         {
3925           "datasource": {
3926             "type": "prometheus",
3927             "uid": "PBFA97CFB590B2093"
3928           },
3929           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3930           "fieldConfig": {
3931             "defaults": {
3932               "color": {
3933                 "mode": "palette-classic"
3934               },
3935               "custom": {
3936                 "axisLabel": "",
3937                 "axisPlacement": "auto",
3938                 "barAlignment": 0,
3939                 "drawStyle": "line",
3940                 "fillOpacity": 10,
3941                 "gradientMode": "none",
3942                 "hideFrom": {
3943                   "legend": false,
3944                   "tooltip": false,
3945                   "viz": false
3946                 },
3947                 "lineInterpolation": "linear",
3948                 "lineWidth": 1,
3949                 "pointSize": 5,
3950                 "scaleDistribution": {
3951                   "type": "linear"
3952                 },
3953                 "showPoints": "never",
3954                 "spanNulls": true,
3955                 "stacking": {
3956                   "group": "A",
3957                   "mode": "none"
3958                 },
3959                 "thresholdsStyle": {
3960                   "mode": "off"
3961                 }
3962               },
3963               "links": [],
3964               "mappings": [],
3965               "thresholds": {
3966                 "mode": "absolute",
3967                 "steps": [
3968                   {
3969                     "color": "green"
3970                   },
3971                   {
3972                     "color": "red",
3973                     "value": 80
3974                   }
3975                 ]
3976               },
3977               "unit": "ms"
3978             },
3979             "overrides": []
3980           },
3981           "gridPos": {
3982             "h": 9,
3983             "w": 12,
3984             "x": 12,
3985             "y": 30
3986           },
3987           "id": 70,
3988           "options": {
3989             "legend": {
3990               "calcs": [
3991                 "mean",
3992                 "lastNotNull",
3993                 "max"
3994               ],
3995               "displayMode": "list",
3996               "placement": "bottom",
3997               "showLegend": true
3998             },
3999             "tooltip": {
4000               "mode": "single"
4001             }
4002           },
4003           "pluginVersion": "8.1.3",
4004           "targets": [
4005             {
4006               "datasource": {
4007                 "type": "prometheus",
4008                 "uid": "PBFA97CFB590B2093"
4009               },
4010               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4011               "hide": false,
4012               "legendFormat": "{{instance}} - {{quantile}}",
4013               "refId": "A"
4014             }
4015           ],
4016           "title": "Consumer - LocalTimeMs",
4017           "type": "timeseries"
4018         },
4019         {
4020           "datasource": {
4021             "type": "prometheus",
4022             "uid": "PBFA97CFB590B2093"
4023           },
4024           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4025           "fieldConfig": {
4026             "defaults": {
4027               "color": {
4028                 "mode": "palette-classic"
4029               },
4030               "custom": {
4031                 "axisLabel": "",
4032                 "axisPlacement": "auto",
4033                 "barAlignment": 0,
4034                 "drawStyle": "line",
4035                 "fillOpacity": 10,
4036                 "gradientMode": "none",
4037                 "hideFrom": {
4038                   "legend": false,
4039                   "tooltip": false,
4040                   "viz": false
4041                 },
4042                 "lineInterpolation": "linear",
4043                 "lineWidth": 1,
4044                 "pointSize": 5,
4045                 "scaleDistribution": {
4046                   "type": "linear"
4047                 },
4048                 "showPoints": "never",
4049                 "spanNulls": true,
4050                 "stacking": {
4051                   "group": "A",
4052                   "mode": "none"
4053                 },
4054                 "thresholdsStyle": {
4055                   "mode": "off"
4056                 }
4057               },
4058               "links": [],
4059               "mappings": [],
4060               "thresholds": {
4061                 "mode": "absolute",
4062                 "steps": [
4063                   {
4064                     "color": "green"
4065                   },
4066                   {
4067                     "color": "red",
4068                     "value": 80
4069                   }
4070                 ]
4071               },
4072               "unit": "ms"
4073             },
4074             "overrides": []
4075           },
4076           "gridPos": {
4077             "h": 9,
4078             "w": 8,
4079             "x": 0,
4080             "y": 39
4081           },
4082           "id": 71,
4083           "options": {
4084             "legend": {
4085               "calcs": [
4086                 "mean",
4087                 "lastNotNull",
4088                 "max"
4089               ],
4090               "displayMode": "table",
4091               "placement": "bottom",
4092               "showLegend": true
4093             },
4094             "tooltip": {
4095               "mode": "single"
4096             }
4097           },
4098           "pluginVersion": "8.1.3",
4099           "targets": [
4100             {
4101               "datasource": {
4102                 "type": "prometheus",
4103                 "uid": "PBFA97CFB590B2093"
4104               },
4105               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4106               "hide": false,
4107               "legendFormat": "{{instance}} - {{quantile}}",
4108               "refId": "A"
4109             }
4110           ],
4111           "title": "Consumer - RemoteTimeMs",
4112           "type": "timeseries"
4113         },
4114         {
4115           "datasource": {
4116             "type": "prometheus",
4117             "uid": "PBFA97CFB590B2093"
4118           },
4119           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4120           "fieldConfig": {
4121             "defaults": {
4122               "color": {
4123                 "mode": "palette-classic"
4124               },
4125               "custom": {
4126                 "axisLabel": "",
4127                 "axisPlacement": "auto",
4128                 "barAlignment": 0,
4129                 "drawStyle": "line",
4130                 "fillOpacity": 10,
4131                 "gradientMode": "none",
4132                 "hideFrom": {
4133                   "legend": false,
4134                   "tooltip": false,
4135                   "viz": false
4136                 },
4137                 "lineInterpolation": "linear",
4138                 "lineWidth": 1,
4139                 "pointSize": 5,
4140                 "scaleDistribution": {
4141                   "type": "linear"
4142                 },
4143                 "showPoints": "never",
4144                 "spanNulls": true,
4145                 "stacking": {
4146                   "group": "A",
4147                   "mode": "none"
4148                 },
4149                 "thresholdsStyle": {
4150                   "mode": "off"
4151                 }
4152               },
4153               "links": [],
4154               "mappings": [],
4155               "min": 0,
4156               "thresholds": {
4157                 "mode": "absolute",
4158                 "steps": [
4159                   {
4160                     "color": "green"
4161                   },
4162                   {
4163                     "color": "red",
4164                     "value": 80
4165                   }
4166                 ]
4167               },
4168               "unit": "ms"
4169             },
4170             "overrides": []
4171           },
4172           "gridPos": {
4173             "h": 9,
4174             "w": 8,
4175             "x": 8,
4176             "y": 39
4177           },
4178           "id": 72,
4179           "options": {
4180             "legend": {
4181               "calcs": [
4182                 "mean",
4183                 "lastNotNull",
4184                 "max"
4185               ],
4186               "displayMode": "table",
4187               "placement": "bottom",
4188               "showLegend": true
4189             },
4190             "tooltip": {
4191               "mode": "single"
4192             }
4193           },
4194           "pluginVersion": "8.1.3",
4195           "targets": [
4196             {
4197               "datasource": {
4198                 "type": "prometheus",
4199                 "uid": "PBFA97CFB590B2093"
4200               },
4201               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4202               "hide": false,
4203               "legendFormat": "{{instance}} - {{quantile}}",
4204               "refId": "A"
4205             }
4206           ],
4207           "title": "Consumer - ResponseQueueTimeMs",
4208           "type": "timeseries"
4209         },
4210         {
4211           "datasource": {
4212             "type": "prometheus",
4213             "uid": "PBFA97CFB590B2093"
4214           },
4215           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4216           "fieldConfig": {
4217             "defaults": {
4218               "color": {
4219                 "mode": "palette-classic"
4220               },
4221               "custom": {
4222                 "axisLabel": "",
4223                 "axisPlacement": "auto",
4224                 "barAlignment": 0,
4225                 "drawStyle": "line",
4226                 "fillOpacity": 10,
4227                 "gradientMode": "none",
4228                 "hideFrom": {
4229                   "legend": false,
4230                   "tooltip": false,
4231                   "viz": false
4232                 },
4233                 "lineInterpolation": "linear",
4234                 "lineWidth": 1,
4235                 "pointSize": 5,
4236                 "scaleDistribution": {
4237                   "type": "linear"
4238                 },
4239                 "showPoints": "never",
4240                 "spanNulls": true,
4241                 "stacking": {
4242                   "group": "A",
4243                   "mode": "none"
4244                 },
4245                 "thresholdsStyle": {
4246                   "mode": "off"
4247                 }
4248               },
4249               "links": [],
4250               "mappings": [],
4251               "min": 0,
4252               "thresholds": {
4253                 "mode": "absolute",
4254                 "steps": [
4255                   {
4256                     "color": "green"
4257                   },
4258                   {
4259                     "color": "red",
4260                     "value": 80
4261                   }
4262                 ]
4263               },
4264               "unit": "ms"
4265             },
4266             "overrides": []
4267           },
4268           "gridPos": {
4269             "h": 9,
4270             "w": 8,
4271             "x": 16,
4272             "y": 39
4273           },
4274           "id": 73,
4275           "options": {
4276             "legend": {
4277               "calcs": [
4278                 "mean",
4279                 "lastNotNull",
4280                 "max"
4281               ],
4282               "displayMode": "table",
4283               "placement": "bottom",
4284               "showLegend": true
4285             },
4286             "tooltip": {
4287               "mode": "single"
4288             }
4289           },
4290           "pluginVersion": "8.1.3",
4291           "targets": [
4292             {
4293               "datasource": {
4294                 "type": "prometheus",
4295                 "uid": "PBFA97CFB590B2093"
4296               },
4297               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4298               "hide": false,
4299               "legendFormat": "{{instance}} - {{quantile}}",
4300               "refId": "A"
4301             }
4302           ],
4303           "title": "Consumer - ResponseSendTimeMs",
4304           "type": "timeseries"
4305         }
4306       ],
4307       "targets": [
4308         {
4309           "datasource": {
4310             "type": "datasource",
4311             "uid": "grafana"
4312           },
4313           "refId": "A"
4314         }
4315       ],
4316       "title": "Consumer Performance",
4317       "type": "row"
4318     },
4319     {
4320       "collapsed": true,
4321       "datasource": {
4322         "type": "datasource",
4323         "uid": "grafana"
4324       },
4325       "gridPos": {
4326         "h": 1,
4327         "w": 24,
4328         "x": 0,
4329         "y": 70
4330       },
4331       "id": 66,
4332       "panels": [
4333         {
4334           "datasource": {
4335             "type": "prometheus",
4336             "uid": "PBFA97CFB590B2093"
4337           },
4338           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4339           "fieldConfig": {
4340             "defaults": {
4341               "color": {
4342                 "mode": "palette-classic"
4343               },
4344               "custom": {
4345                 "axisLabel": "",
4346                 "axisPlacement": "auto",
4347                 "barAlignment": 0,
4348                 "drawStyle": "line",
4349                 "fillOpacity": 10,
4350                 "gradientMode": "none",
4351                 "hideFrom": {
4352                   "legend": false,
4353                   "tooltip": false,
4354                   "viz": false
4355                 },
4356                 "lineInterpolation": "linear",
4357                 "lineWidth": 1,
4358                 "pointSize": 5,
4359                 "scaleDistribution": {
4360                   "type": "linear"
4361                 },
4362                 "showPoints": "never",
4363                 "spanNulls": true,
4364                 "stacking": {
4365                   "group": "A",
4366                   "mode": "none"
4367                 },
4368                 "thresholdsStyle": {
4369                   "mode": "off"
4370                 }
4371               },
4372               "links": [],
4373               "mappings": [],
4374               "min": 0,
4375               "thresholds": {
4376                 "mode": "absolute",
4377                 "steps": [
4378                   {
4379                     "color": "green"
4380                   },
4381                   {
4382                     "color": "red",
4383                     "value": 80
4384                   }
4385                 ]
4386               },
4387               "unit": "ms"
4388             },
4389             "overrides": []
4390           },
4391           "gridPos": {
4392             "h": 9,
4393             "w": 12,
4394             "x": 0,
4395             "y": 31
4396           },
4397           "id": 74,
4398           "options": {
4399             "legend": {
4400               "calcs": [
4401                 "mean",
4402                 "lastNotNull",
4403                 "max"
4404               ],
4405               "displayMode": "list",
4406               "placement": "bottom",
4407               "showLegend": true
4408             },
4409             "tooltip": {
4410               "mode": "single"
4411             }
4412           },
4413           "pluginVersion": "8.1.3",
4414           "targets": [
4415             {
4416               "datasource": {
4417                 "type": "prometheus",
4418                 "uid": "PBFA97CFB590B2093"
4419               },
4420               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4421               "hide": false,
4422               "legendFormat": "{{instance}} - {{quantile}}",
4423               "refId": "A"
4424             }
4425           ],
4426           "title": "FetchFollower - RequestQueueTimeMs",
4427           "type": "timeseries"
4428         },
4429         {
4430           "datasource": {
4431             "type": "prometheus",
4432             "uid": "PBFA97CFB590B2093"
4433           },
4434           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4435           "fieldConfig": {
4436             "defaults": {
4437               "color": {
4438                 "mode": "palette-classic"
4439               },
4440               "custom": {
4441                 "axisLabel": "",
4442                 "axisPlacement": "auto",
4443                 "barAlignment": 0,
4444                 "drawStyle": "line",
4445                 "fillOpacity": 10,
4446                 "gradientMode": "none",
4447                 "hideFrom": {
4448                   "legend": false,
4449                   "tooltip": false,
4450                   "viz": false
4451                 },
4452                 "lineInterpolation": "linear",
4453                 "lineWidth": 1,
4454                 "pointSize": 5,
4455                 "scaleDistribution": {
4456                   "type": "linear"
4457                 },
4458                 "showPoints": "never",
4459                 "spanNulls": true,
4460                 "stacking": {
4461                   "group": "A",
4462                   "mode": "none"
4463                 },
4464                 "thresholdsStyle": {
4465                   "mode": "off"
4466                 }
4467               },
4468               "links": [],
4469               "mappings": [],
4470               "min": 0,
4471               "thresholds": {
4472                 "mode": "absolute",
4473                 "steps": [
4474                   {
4475                     "color": "green"
4476                   },
4477                   {
4478                     "color": "red",
4479                     "value": 80
4480                   }
4481                 ]
4482               },
4483               "unit": "ms"
4484             },
4485             "overrides": []
4486           },
4487           "gridPos": {
4488             "h": 9,
4489             "w": 12,
4490             "x": 12,
4491             "y": 31
4492           },
4493           "id": 75,
4494           "options": {
4495             "legend": {
4496               "calcs": [
4497                 "mean",
4498                 "lastNotNull",
4499                 "max"
4500               ],
4501               "displayMode": "list",
4502               "placement": "bottom",
4503               "showLegend": true
4504             },
4505             "tooltip": {
4506               "mode": "single"
4507             }
4508           },
4509           "pluginVersion": "8.1.3",
4510           "targets": [
4511             {
4512               "datasource": {
4513                 "type": "prometheus",
4514                 "uid": "PBFA97CFB590B2093"
4515               },
4516               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4517               "hide": false,
4518               "legendFormat": "{{instance}} - {{quantile}}",
4519               "refId": "A"
4520             }
4521           ],
4522           "title": "FetchFollower - LocalTimeMs",
4523           "type": "timeseries"
4524         },
4525         {
4526           "datasource": {
4527             "type": "prometheus",
4528             "uid": "PBFA97CFB590B2093"
4529           },
4530           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4531           "fieldConfig": {
4532             "defaults": {
4533               "color": {
4534                 "mode": "palette-classic"
4535               },
4536               "custom": {
4537                 "axisLabel": "",
4538                 "axisPlacement": "auto",
4539                 "barAlignment": 0,
4540                 "drawStyle": "line",
4541                 "fillOpacity": 10,
4542                 "gradientMode": "none",
4543                 "hideFrom": {
4544                   "legend": false,
4545                   "tooltip": false,
4546                   "viz": false
4547                 },
4548                 "lineInterpolation": "linear",
4549                 "lineWidth": 1,
4550                 "pointSize": 5,
4551                 "scaleDistribution": {
4552                   "type": "linear"
4553                 },
4554                 "showPoints": "never",
4555                 "spanNulls": true,
4556                 "stacking": {
4557                   "group": "A",
4558                   "mode": "none"
4559                 },
4560                 "thresholdsStyle": {
4561                   "mode": "off"
4562                 }
4563               },
4564               "links": [],
4565               "mappings": [],
4566               "thresholds": {
4567                 "mode": "absolute",
4568                 "steps": [
4569                   {
4570                     "color": "green"
4571                   },
4572                   {
4573                     "color": "red",
4574                     "value": 80
4575                   }
4576                 ]
4577               },
4578               "unit": "ms"
4579             },
4580             "overrides": []
4581           },
4582           "gridPos": {
4583             "h": 9,
4584             "w": 8,
4585             "x": 0,
4586             "y": 40
4587           },
4588           "id": 76,
4589           "options": {
4590             "legend": {
4591               "calcs": [
4592                 "mean",
4593                 "lastNotNull",
4594                 "max"
4595               ],
4596               "displayMode": "table",
4597               "placement": "bottom",
4598               "showLegend": true
4599             },
4600             "tooltip": {
4601               "mode": "single"
4602             }
4603           },
4604           "pluginVersion": "8.1.3",
4605           "targets": [
4606             {
4607               "datasource": {
4608                 "type": "prometheus",
4609                 "uid": "PBFA97CFB590B2093"
4610               },
4611               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4612               "hide": false,
4613               "legendFormat": "{{instance}} - {{quantile}}",
4614               "refId": "A"
4615             }
4616           ],
4617           "title": "FetchFollower - RemoteTimeMs",
4618           "type": "timeseries"
4619         },
4620         {
4621           "datasource": {
4622             "type": "prometheus",
4623             "uid": "PBFA97CFB590B2093"
4624           },
4625           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4626           "fieldConfig": {
4627             "defaults": {
4628               "color": {
4629                 "mode": "palette-classic"
4630               },
4631               "custom": {
4632                 "axisLabel": "",
4633                 "axisPlacement": "auto",
4634                 "barAlignment": 0,
4635                 "drawStyle": "line",
4636                 "fillOpacity": 10,
4637                 "gradientMode": "none",
4638                 "hideFrom": {
4639                   "legend": false,
4640                   "tooltip": false,
4641                   "viz": false
4642                 },
4643                 "lineInterpolation": "linear",
4644                 "lineWidth": 1,
4645                 "pointSize": 5,
4646                 "scaleDistribution": {
4647                   "type": "linear"
4648                 },
4649                 "showPoints": "never",
4650                 "spanNulls": true,
4651                 "stacking": {
4652                   "group": "A",
4653                   "mode": "none"
4654                 },
4655                 "thresholdsStyle": {
4656                   "mode": "off"
4657                 }
4658               },
4659               "links": [],
4660               "mappings": [],
4661               "thresholds": {
4662                 "mode": "absolute",
4663                 "steps": [
4664                   {
4665                     "color": "green"
4666                   },
4667                   {
4668                     "color": "red",
4669                     "value": 80
4670                   }
4671                 ]
4672               },
4673               "unit": "ms"
4674             },
4675             "overrides": []
4676           },
4677           "gridPos": {
4678             "h": 9,
4679             "w": 8,
4680             "x": 8,
4681             "y": 40
4682           },
4683           "id": 77,
4684           "options": {
4685             "legend": {
4686               "calcs": [
4687                 "mean",
4688                 "lastNotNull",
4689                 "max"
4690               ],
4691               "displayMode": "table",
4692               "placement": "bottom",
4693               "showLegend": true
4694             },
4695             "tooltip": {
4696               "mode": "single"
4697             }
4698           },
4699           "pluginVersion": "8.1.3",
4700           "targets": [
4701             {
4702               "datasource": {
4703                 "type": "prometheus",
4704                 "uid": "PBFA97CFB590B2093"
4705               },
4706               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4707               "hide": false,
4708               "legendFormat": "{{instance}} - {{quantile}}",
4709               "refId": "A"
4710             }
4711           ],
4712           "title": "FetchFollower - ResponseQueueTimeMs",
4713           "type": "timeseries"
4714         },
4715         {
4716           "datasource": {
4717             "type": "prometheus",
4718             "uid": "PBFA97CFB590B2093"
4719           },
4720           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4721           "fieldConfig": {
4722             "defaults": {
4723               "color": {
4724                 "mode": "palette-classic"
4725               },
4726               "custom": {
4727                 "axisLabel": "",
4728                 "axisPlacement": "auto",
4729                 "barAlignment": 0,
4730                 "drawStyle": "line",
4731                 "fillOpacity": 10,
4732                 "gradientMode": "none",
4733                 "hideFrom": {
4734                   "legend": false,
4735                   "tooltip": false,
4736                   "viz": false
4737                 },
4738                 "lineInterpolation": "linear",
4739                 "lineWidth": 1,
4740                 "pointSize": 5,
4741                 "scaleDistribution": {
4742                   "type": "linear"
4743                 },
4744                 "showPoints": "never",
4745                 "spanNulls": true,
4746                 "stacking": {
4747                   "group": "A",
4748                   "mode": "none"
4749                 },
4750                 "thresholdsStyle": {
4751                   "mode": "off"
4752                 }
4753               },
4754               "links": [],
4755               "mappings": [],
4756               "min": 0,
4757               "thresholds": {
4758                 "mode": "absolute",
4759                 "steps": [
4760                   {
4761                     "color": "green"
4762                   },
4763                   {
4764                     "color": "red",
4765                     "value": 80
4766                   }
4767                 ]
4768               },
4769               "unit": "ms"
4770             },
4771             "overrides": []
4772           },
4773           "gridPos": {
4774             "h": 9,
4775             "w": 8,
4776             "x": 16,
4777             "y": 40
4778           },
4779           "id": 78,
4780           "options": {
4781             "legend": {
4782               "calcs": [
4783                 "mean",
4784                 "lastNotNull",
4785                 "max"
4786               ],
4787               "displayMode": "table",
4788               "placement": "bottom",
4789               "showLegend": true
4790             },
4791             "tooltip": {
4792               "mode": "single"
4793             }
4794           },
4795           "pluginVersion": "8.1.3",
4796           "targets": [
4797             {
4798               "datasource": {
4799                 "type": "prometheus",
4800                 "uid": "PBFA97CFB590B2093"
4801               },
4802               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4803               "hide": false,
4804               "legendFormat": "{{instance}} - {{quantile}}",
4805               "refId": "A"
4806             }
4807           ],
4808           "title": "FetchFollower - ResponseSendTimeMs",
4809           "type": "timeseries"
4810         }
4811       ],
4812       "targets": [
4813         {
4814           "datasource": {
4815             "type": "datasource",
4816             "uid": "grafana"
4817           },
4818           "refId": "A"
4819         }
4820       ],
4821       "title": "Fetch Follower Performance",
4822       "type": "row"
4823     },
4824     {
4825       "collapsed": true,
4826       "datasource": {
4827         "type": "datasource",
4828         "uid": "grafana"
4829       },
4830       "gridPos": {
4831         "h": 1,
4832         "w": 24,
4833         "x": 0,
4834         "y": 71
4835       },
4836       "id": 102,
4837       "panels": [
4838         {
4839           "datasource": {
4840             "type": "prometheus",
4841             "uid": "PBFA97CFB590B2093"
4842           },
4843           "fieldConfig": {
4844             "defaults": {
4845               "color": {
4846                 "mode": "palette-classic"
4847               },
4848               "custom": {
4849                 "axisLabel": "",
4850                 "axisPlacement": "auto",
4851                 "barAlignment": 0,
4852                 "drawStyle": "line",
4853                 "fillOpacity": 10,
4854                 "gradientMode": "none",
4855                 "hideFrom": {
4856                   "legend": false,
4857                   "tooltip": false,
4858                   "viz": false
4859                 },
4860                 "lineInterpolation": "linear",
4861                 "lineWidth": 1,
4862                 "pointSize": 5,
4863                 "scaleDistribution": {
4864                   "type": "linear"
4865                 },
4866                 "showPoints": "never",
4867                 "spanNulls": true,
4868                 "stacking": {
4869                   "group": "A",
4870                   "mode": "none"
4871                 },
4872                 "thresholdsStyle": {
4873                   "mode": "off"
4874                 }
4875               },
4876               "links": [],
4877               "mappings": [],
4878               "thresholds": {
4879                 "mode": "absolute",
4880                 "steps": [
4881                   {
4882                     "color": "green"
4883                   },
4884                   {
4885                     "color": "red",
4886                     "value": 80
4887                   }
4888                 ]
4889               },
4890               "unit": "short"
4891             },
4892             "overrides": []
4893           },
4894           "gridPos": {
4895             "h": 8,
4896             "w": 12,
4897             "x": 0,
4898             "y": 32
4899           },
4900           "id": 98,
4901           "options": {
4902             "legend": {
4903               "calcs": [
4904                 "mean",
4905                 "lastNotNull",
4906                 "max"
4907               ],
4908               "displayMode": "table",
4909               "placement": "bottom",
4910               "showLegend": true
4911             },
4912             "tooltip": {
4913               "mode": "single"
4914             }
4915           },
4916           "pluginVersion": "8.1.3",
4917           "targets": [
4918             {
4919               "datasource": {
4920                 "type": "prometheus",
4921                 "uid": "PBFA97CFB590B2093"
4922               },
4923               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4924               "interval": "",
4925               "legendFormat": "{{listener}}",
4926               "refId": "A"
4927             }
4928           ],
4929           "title": "Connections count per listener",
4930           "type": "timeseries"
4931         },
4932         {
4933           "datasource": {
4934             "type": "prometheus",
4935             "uid": "PBFA97CFB590B2093"
4936           },
4937           "fieldConfig": {
4938             "defaults": {
4939               "color": {
4940                 "mode": "palette-classic"
4941               },
4942               "custom": {
4943                 "axisLabel": "",
4944                 "axisPlacement": "auto",
4945                 "barAlignment": 0,
4946                 "drawStyle": "line",
4947                 "fillOpacity": 10,
4948                 "gradientMode": "none",
4949                 "hideFrom": {
4950                   "legend": false,
4951                   "tooltip": false,
4952                   "viz": false
4953                 },
4954                 "lineInterpolation": "linear",
4955                 "lineWidth": 1,
4956                 "pointSize": 5,
4957                 "scaleDistribution": {
4958                   "type": "linear"
4959                 },
4960                 "showPoints": "never",
4961                 "spanNulls": true,
4962                 "stacking": {
4963                   "group": "A",
4964                   "mode": "none"
4965                 },
4966                 "thresholdsStyle": {
4967                   "mode": "off"
4968                 }
4969               },
4970               "links": [],
4971               "mappings": [],
4972               "thresholds": {
4973                 "mode": "absolute",
4974                 "steps": [
4975                   {
4976                     "color": "green"
4977                   },
4978                   {
4979                     "color": "red",
4980                     "value": 80
4981                   }
4982                 ]
4983               },
4984               "unit": "short"
4985             },
4986             "overrides": []
4987           },
4988           "gridPos": {
4989             "h": 8,
4990             "w": 12,
4991             "x": 12,
4992             "y": 32
4993           },
4994           "id": 100,
4995           "options": {
4996             "legend": {
4997               "calcs": [
4998                 "mean",
4999                 "lastNotNull",
5000                 "max"
5001               ],
5002               "displayMode": "table",
5003               "placement": "bottom",
5004               "showLegend": true
5005             },
5006             "tooltip": {
5007               "mode": "single"
5008             }
5009           },
5010           "pluginVersion": "8.1.3",
5011           "targets": [
5012             {
5013               "datasource": {
5014                 "type": "prometheus",
5015                 "uid": "PBFA97CFB590B2093"
5016               },
5017               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5018               "interval": "",
5019               "legendFormat": "{{instance}}",
5020               "refId": "A"
5021             }
5022           ],
5023           "title": "Connections count per broker",
5024           "type": "timeseries"
5025         },
5026         {
5027           "datasource": {
5028             "type": "prometheus",
5029             "uid": "PBFA97CFB590B2093"
5030           },
5031           "fieldConfig": {
5032             "defaults": {
5033               "color": {
5034                 "mode": "palette-classic"
5035               },
5036               "custom": {
5037                 "axisLabel": "",
5038                 "axisPlacement": "auto",
5039                 "barAlignment": 0,
5040                 "drawStyle": "line",
5041                 "fillOpacity": 10,
5042                 "gradientMode": "none",
5043                 "hideFrom": {
5044                   "legend": false,
5045                   "tooltip": false,
5046                   "viz": false
5047                 },
5048                 "lineInterpolation": "linear",
5049                 "lineWidth": 1,
5050                 "pointSize": 5,
5051                 "scaleDistribution": {
5052                   "type": "linear"
5053                 },
5054                 "showPoints": "never",
5055                 "spanNulls": true,
5056                 "stacking": {
5057                   "group": "A",
5058                   "mode": "none"
5059                 },
5060                 "thresholdsStyle": {
5061                   "mode": "off"
5062                 }
5063               },
5064               "links": [],
5065               "mappings": [],
5066               "thresholds": {
5067                 "mode": "absolute",
5068                 "steps": [
5069                   {
5070                     "color": "green"
5071                   },
5072                   {
5073                     "color": "red",
5074                     "value": 80
5075                   }
5076                 ]
5077               },
5078               "unit": "short"
5079             },
5080             "overrides": []
5081           },
5082           "gridPos": {
5083             "h": 8,
5084             "w": 12,
5085             "x": 0,
5086             "y": 40
5087           },
5088           "id": 104,
5089           "options": {
5090             "legend": {
5091               "calcs": [
5092                 "mean",
5093                 "lastNotNull",
5094                 "max"
5095               ],
5096               "displayMode": "table",
5097               "placement": "bottom",
5098               "showLegend": true
5099             },
5100             "tooltip": {
5101               "mode": "single"
5102             }
5103           },
5104           "pluginVersion": "8.1.3",
5105           "targets": [
5106             {
5107               "datasource": {
5108                 "type": "prometheus",
5109                 "uid": "PBFA97CFB590B2093"
5110               },
5111               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5112               "interval": "",
5113               "legendFormat": "{{listener}}",
5114               "refId": "A"
5115             }
5116           ],
5117           "title": "Connections creation rate per listener",
5118           "type": "timeseries"
5119         },
5120         {
5121           "datasource": {
5122             "type": "prometheus",
5123             "uid": "PBFA97CFB590B2093"
5124           },
5125           "fieldConfig": {
5126             "defaults": {
5127               "color": {
5128                 "mode": "palette-classic"
5129               },
5130               "custom": {
5131                 "axisLabel": "",
5132                 "axisPlacement": "auto",
5133                 "barAlignment": 0,
5134                 "drawStyle": "line",
5135                 "fillOpacity": 10,
5136                 "gradientMode": "none",
5137                 "hideFrom": {
5138                   "legend": false,
5139                   "tooltip": false,
5140                   "viz": false
5141                 },
5142                 "lineInterpolation": "linear",
5143                 "lineWidth": 1,
5144                 "pointSize": 5,
5145                 "scaleDistribution": {
5146                   "type": "linear"
5147                 },
5148                 "showPoints": "never",
5149                 "spanNulls": true,
5150                 "stacking": {
5151                   "group": "A",
5152                   "mode": "none"
5153                 },
5154                 "thresholdsStyle": {
5155                   "mode": "off"
5156                 }
5157               },
5158               "links": [],
5159               "mappings": [],
5160               "thresholds": {
5161                 "mode": "absolute",
5162                 "steps": [
5163                   {
5164                     "color": "green"
5165                   },
5166                   {
5167                     "color": "red",
5168                     "value": 80
5169                   }
5170                 ]
5171               },
5172               "unit": "short"
5173             },
5174             "overrides": []
5175           },
5176           "gridPos": {
5177             "h": 8,
5178             "w": 12,
5179             "x": 12,
5180             "y": 40
5181           },
5182           "id": 106,
5183           "options": {
5184             "legend": {
5185               "calcs": [
5186                 "mean",
5187                 "lastNotNull",
5188                 "max"
5189               ],
5190               "displayMode": "table",
5191               "placement": "bottom",
5192               "showLegend": true
5193             },
5194             "tooltip": {
5195               "mode": "single"
5196             }
5197           },
5198           "pluginVersion": "8.1.3",
5199           "targets": [
5200             {
5201               "datasource": {
5202                 "type": "prometheus",
5203                 "uid": "PBFA97CFB590B2093"
5204               },
5205               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5206               "interval": "",
5207               "legendFormat": "{{instance}}",
5208               "refId": "A"
5209             }
5210           ],
5211           "title": "Connections creation rate per instance",
5212           "type": "timeseries"
5213         },
5214         {
5215           "datasource": {
5216             "type": "prometheus",
5217             "uid": "PBFA97CFB590B2093"
5218           },
5219           "fieldConfig": {
5220             "defaults": {
5221               "color": {
5222                 "mode": "palette-classic"
5223               },
5224               "custom": {
5225                 "axisLabel": "",
5226                 "axisPlacement": "auto",
5227                 "barAlignment": 0,
5228                 "drawStyle": "line",
5229                 "fillOpacity": 10,
5230                 "gradientMode": "none",
5231                 "hideFrom": {
5232                   "legend": false,
5233                   "tooltip": false,
5234                   "viz": false
5235                 },
5236                 "lineInterpolation": "linear",
5237                 "lineWidth": 1,
5238                 "pointSize": 5,
5239                 "scaleDistribution": {
5240                   "type": "linear"
5241                 },
5242                 "showPoints": "never",
5243                 "spanNulls": true,
5244                 "stacking": {
5245                   "group": "A",
5246                   "mode": "none"
5247                 },
5248                 "thresholdsStyle": {
5249                   "mode": "off"
5250                 }
5251               },
5252               "links": [],
5253               "mappings": [],
5254               "thresholds": {
5255                 "mode": "absolute",
5256                 "steps": [
5257                   {
5258                     "color": "green"
5259                   },
5260                   {
5261                     "color": "red",
5262                     "value": 80
5263                   }
5264                 ]
5265               },
5266               "unit": "short"
5267             },
5268             "overrides": []
5269           },
5270           "gridPos": {
5271             "h": 8,
5272             "w": 12,
5273             "x": 0,
5274             "y": 48
5275           },
5276           "id": 108,
5277           "options": {
5278             "legend": {
5279               "calcs": [
5280                 "mean",
5281                 "lastNotNull",
5282                 "max"
5283               ],
5284               "displayMode": "table",
5285               "placement": "bottom",
5286               "showLegend": true
5287             },
5288             "tooltip": {
5289               "mode": "single"
5290             }
5291           },
5292           "pluginVersion": "8.1.3",
5293           "targets": [
5294             {
5295               "datasource": {
5296                 "type": "prometheus",
5297                 "uid": "PBFA97CFB590B2093"
5298               },
5299               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5300               "interval": "",
5301               "legendFormat": "{{listener}}",
5302               "refId": "A"
5303             }
5304           ],
5305           "title": "Connections close rate per listener",
5306           "type": "timeseries"
5307         },
5308         {
5309           "datasource": {
5310             "type": "prometheus",
5311             "uid": "PBFA97CFB590B2093"
5312           },
5313           "fieldConfig": {
5314             "defaults": {
5315               "color": {
5316                 "mode": "palette-classic"
5317               },
5318               "custom": {
5319                 "axisLabel": "",
5320                 "axisPlacement": "auto",
5321                 "barAlignment": 0,
5322                 "drawStyle": "line",
5323                 "fillOpacity": 10,
5324                 "gradientMode": "none",
5325                 "hideFrom": {
5326                   "legend": false,
5327                   "tooltip": false,
5328                   "viz": false
5329                 },
5330                 "lineInterpolation": "linear",
5331                 "lineWidth": 1,
5332                 "pointSize": 5,
5333                 "scaleDistribution": {
5334                   "type": "linear"
5335                 },
5336                 "showPoints": "never",
5337                 "spanNulls": true,
5338                 "stacking": {
5339                   "group": "A",
5340                   "mode": "none"
5341                 },
5342                 "thresholdsStyle": {
5343                   "mode": "off"
5344                 }
5345               },
5346               "links": [],
5347               "mappings": [],
5348               "thresholds": {
5349                 "mode": "absolute",
5350                 "steps": [
5351                   {
5352                     "color": "green"
5353                   },
5354                   {
5355                     "color": "red",
5356                     "value": 80
5357                   }
5358                 ]
5359               },
5360               "unit": "short"
5361             },
5362             "overrides": []
5363           },
5364           "gridPos": {
5365             "h": 8,
5366             "w": 12,
5367             "x": 12,
5368             "y": 48
5369           },
5370           "id": 110,
5371           "options": {
5372             "legend": {
5373               "calcs": [
5374                 "mean",
5375                 "lastNotNull",
5376                 "max"
5377               ],
5378               "displayMode": "table",
5379               "placement": "bottom",
5380               "showLegend": true
5381             },
5382             "tooltip": {
5383               "mode": "single"
5384             }
5385           },
5386           "pluginVersion": "8.1.3",
5387           "targets": [
5388             {
5389               "datasource": {
5390                 "type": "prometheus",
5391                 "uid": "PBFA97CFB590B2093"
5392               },
5393               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5394               "interval": "",
5395               "legendFormat": "{{instance}}",
5396               "refId": "A"
5397             }
5398           ],
5399           "title": "Connections close rate per instance",
5400           "type": "timeseries"
5401         },
5402         {
5403           "datasource": {
5404             "type": "prometheus",
5405             "uid": "PBFA97CFB590B2093"
5406           },
5407           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5408           "fieldConfig": {
5409             "defaults": {
5410               "color": {
5411                 "mode": "palette-classic"
5412               },
5413               "custom": {
5414                 "axisLabel": "",
5415                 "axisPlacement": "auto",
5416                 "barAlignment": 0,
5417                 "drawStyle": "line",
5418                 "fillOpacity": 10,
5419                 "gradientMode": "none",
5420                 "hideFrom": {
5421                   "legend": false,
5422                   "tooltip": false,
5423                   "viz": false
5424                 },
5425                 "lineInterpolation": "linear",
5426                 "lineWidth": 1,
5427                 "pointSize": 5,
5428                 "scaleDistribution": {
5429                   "type": "linear"
5430                 },
5431                 "showPoints": "never",
5432                 "spanNulls": true,
5433                 "stacking": {
5434                   "group": "A",
5435                   "mode": "none"
5436                 },
5437                 "thresholdsStyle": {
5438                   "mode": "off"
5439                 }
5440               },
5441               "links": [],
5442               "mappings": [],
5443               "min": 0,
5444               "thresholds": {
5445                 "mode": "absolute",
5446                 "steps": [
5447                   {
5448                     "color": "green"
5449                   },
5450                   {
5451                     "color": "red",
5452                     "value": 80
5453                   }
5454                 ]
5455               },
5456               "unit": "percent"
5457             },
5458             "overrides": []
5459           },
5460           "gridPos": {
5461             "h": 8,
5462             "w": 12,
5463             "x": 0,
5464             "y": 56
5465           },
5466           "id": 112,
5467           "options": {
5468             "legend": {
5469               "calcs": [
5470                 "mean",
5471                 "lastNotNull",
5472                 "max"
5473               ],
5474               "displayMode": "table",
5475               "placement": "bottom",
5476               "showLegend": true
5477             },
5478             "tooltip": {
5479               "mode": "single"
5480             }
5481           },
5482           "pluginVersion": "8.1.3",
5483           "targets": [
5484             {
5485               "datasource": {
5486                 "type": "prometheus",
5487                 "uid": "PBFA97CFB590B2093"
5488               },
5489               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5490               "interval": "",
5491               "legendFormat": "{{instance}} - {{listener}}",
5492               "refId": "A"
5493             }
5494           ],
5495           "title": "Acceptor Blocked Percentage",
5496           "type": "timeseries"
5497         },
5498         {
5499           "datasource": {
5500             "type": "prometheus",
5501             "uid": "PBFA97CFB590B2093"
5502           },
5503           "fieldConfig": {
5504             "defaults": {
5505               "color": {
5506                 "mode": "palette-classic"
5507               },
5508               "custom": {
5509                 "axisLabel": "",
5510                 "axisPlacement": "auto",
5511                 "barAlignment": 0,
5512                 "drawStyle": "line",
5513                 "fillOpacity": 10,
5514                 "gradientMode": "none",
5515                 "hideFrom": {
5516                   "legend": false,
5517                   "tooltip": false,
5518                   "viz": false
5519                 },
5520                 "lineInterpolation": "linear",
5521                 "lineWidth": 1,
5522                 "pointSize": 5,
5523                 "scaleDistribution": {
5524                   "type": "linear"
5525                 },
5526                 "showPoints": "never",
5527                 "spanNulls": true,
5528                 "stacking": {
5529                   "group": "A",
5530                   "mode": "none"
5531                 },
5532                 "thresholdsStyle": {
5533                   "mode": "off"
5534                 }
5535               },
5536               "links": [],
5537               "mappings": [],
5538               "thresholds": {
5539                 "mode": "absolute",
5540                 "steps": [
5541                   {
5542                     "color": "green"
5543                   },
5544                   {
5545                     "color": "red",
5546                     "value": 80
5547                   }
5548                 ]
5549               },
5550               "unit": "short"
5551             },
5552             "overrides": []
5553           },
5554           "gridPos": {
5555             "h": 8,
5556             "w": 12,
5557             "x": 12,
5558             "y": 56
5559           },
5560           "id": 114,
5561           "options": {
5562             "legend": {
5563               "calcs": [
5564                 "mean",
5565                 "lastNotNull",
5566                 "max"
5567               ],
5568               "displayMode": "table",
5569               "placement": "bottom",
5570               "showLegend": true
5571             },
5572             "tooltip": {
5573               "mode": "single"
5574             }
5575           },
5576           "pluginVersion": "8.1.3",
5577           "targets": [
5578             {
5579               "datasource": {
5580                 "type": "prometheus",
5581                 "uid": "PBFA97CFB590B2093"
5582               },
5583               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5584               "interval": "",
5585               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5586               "refId": "A"
5587             }
5588           ],
5589           "title": "Connections per client version",
5590           "type": "timeseries"
5591         }
5592       ],
5593       "targets": [
5594         {
5595           "datasource": {
5596             "type": "datasource",
5597             "uid": "grafana"
5598           },
5599           "refId": "A"
5600         }
5601       ],
5602       "title": "Connections",
5603       "type": "row"
5604     },
5605     {
5606       "collapsed": true,
5607       "datasource": {
5608         "type": "datasource",
5609         "uid": "grafana"
5610       },
5611       "gridPos": {
5612         "h": 1,
5613         "w": 24,
5614         "x": 0,
5615         "y": 72
5616       },
5617       "id": 120,
5618       "panels": [
5619         {
5620           "datasource": {
5621             "type": "prometheus",
5622             "uid": "PBFA97CFB590B2093"
5623           },
5624           "description": "Number of consumer groups per group coordinator",
5625           "fieldConfig": {
5626             "defaults": {
5627               "color": {
5628                 "mode": "palette-classic"
5629               },
5630               "custom": {
5631                 "axisLabel": "",
5632                 "axisPlacement": "auto",
5633                 "barAlignment": 0,
5634                 "drawStyle": "line",
5635                 "fillOpacity": 10,
5636                 "gradientMode": "none",
5637                 "hideFrom": {
5638                   "legend": false,
5639                   "tooltip": false,
5640                   "viz": false
5641                 },
5642                 "lineInterpolation": "linear",
5643                 "lineWidth": 1,
5644                 "pointSize": 5,
5645                 "scaleDistribution": {
5646                   "type": "linear"
5647                 },
5648                 "showPoints": "never",
5649                 "spanNulls": true,
5650                 "stacking": {
5651                   "group": "A",
5652                   "mode": "none"
5653                 },
5654                 "thresholdsStyle": {
5655                   "mode": "off"
5656                 }
5657               },
5658               "links": [],
5659               "mappings": [],
5660               "thresholds": {
5661                 "mode": "absolute",
5662                 "steps": [
5663                   {
5664                     "color": "green"
5665                   },
5666                   {
5667                     "color": "red",
5668                     "value": 80
5669                   }
5670                 ]
5671               },
5672               "unit": "short"
5673             },
5674             "overrides": []
5675           },
5676           "gridPos": {
5677             "h": 8,
5678             "w": 12,
5679             "x": 0,
5680             "y": 33
5681           },
5682           "id": 116,
5683           "options": {
5684             "legend": {
5685               "calcs": [
5686                 "mean",
5687                 "lastNotNull",
5688                 "max"
5689               ],
5690               "displayMode": "table",
5691               "placement": "bottom",
5692               "showLegend": true
5693             },
5694             "tooltip": {
5695               "mode": "single"
5696             }
5697           },
5698           "pluginVersion": "8.1.3",
5699           "targets": [
5700             {
5701               "datasource": {
5702                 "type": "prometheus",
5703                 "uid": "PBFA97CFB590B2093"
5704               },
5705               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5706               "instant": false,
5707               "interval": "",
5708               "legendFormat": "{{instance}}",
5709               "refId": "A"
5710             }
5711           ],
5712           "title": "Consumer groups number per coordinator",
5713           "type": "timeseries"
5714         },
5715         {
5716           "datasource": {
5717             "type": "prometheus",
5718             "uid": "PBFA97CFB590B2093"
5719           },
5720           "description": "Number of consumer group per state",
5721           "fieldConfig": {
5722             "defaults": {
5723               "color": {
5724                 "mode": "palette-classic"
5725               },
5726               "custom": {
5727                 "axisLabel": "",
5728                 "axisPlacement": "auto",
5729                 "barAlignment": 0,
5730                 "drawStyle": "line",
5731                 "fillOpacity": 10,
5732                 "gradientMode": "none",
5733                 "hideFrom": {
5734                   "legend": false,
5735                   "tooltip": false,
5736                   "viz": false
5737                 },
5738                 "lineInterpolation": "linear",
5739                 "lineWidth": 1,
5740                 "pointSize": 5,
5741                 "scaleDistribution": {
5742                   "type": "linear"
5743                 },
5744                 "showPoints": "never",
5745                 "spanNulls": true,
5746                 "stacking": {
5747                   "group": "A",
5748                   "mode": "none"
5749                 },
5750                 "thresholdsStyle": {
5751                   "mode": "off"
5752                 }
5753               },
5754               "links": [],
5755               "mappings": [],
5756               "thresholds": {
5757                 "mode": "absolute",
5758                 "steps": [
5759                   {
5760                     "color": "green"
5761                   },
5762                   {
5763                     "color": "red",
5764                     "value": 80
5765                   }
5766                 ]
5767               },
5768               "unit": "short"
5769             },
5770             "overrides": []
5771           },
5772           "gridPos": {
5773             "h": 8,
5774             "w": 12,
5775             "x": 12,
5776             "y": 33
5777           },
5778           "id": 118,
5779           "options": {
5780             "legend": {
5781               "calcs": [
5782                 "mean",
5783                 "lastNotNull",
5784                 "max"
5785               ],
5786               "displayMode": "table",
5787               "placement": "bottom",
5788               "showLegend": true
5789             },
5790             "tooltip": {
5791               "mode": "single"
5792             }
5793           },
5794           "pluginVersion": "8.1.3",
5795           "targets": [
5796             {
5797               "datasource": {
5798                 "type": "prometheus",
5799                 "uid": "PBFA97CFB590B2093"
5800               },
5801               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5802               "instant": false,
5803               "interval": "",
5804               "legendFormat": "stable",
5805               "refId": "A"
5806             },
5807             {
5808               "datasource": {
5809                 "type": "prometheus",
5810                 "uid": "PBFA97CFB590B2093"
5811               },
5812               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5813               "interval": "",
5814               "legendFormat": "preparing-rebalance",
5815               "refId": "B"
5816             },
5817             {
5818               "datasource": {
5819                 "type": "prometheus",
5820                 "uid": "PBFA97CFB590B2093"
5821               },
5822               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5823               "interval": "",
5824               "legendFormat": "dead",
5825               "refId": "C"
5826             },
5827             {
5828               "datasource": {
5829                 "type": "prometheus",
5830                 "uid": "PBFA97CFB590B2093"
5831               },
5832               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5833               "interval": "",
5834               "legendFormat": "completing-rebalance",
5835               "refId": "D"
5836             },
5837             {
5838               "datasource": {
5839                 "type": "prometheus",
5840                 "uid": "PBFA97CFB590B2093"
5841               },
5842               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5843               "interval": "",
5844               "legendFormat": "empty",
5845               "refId": "E"
5846             }
5847           ],
5848           "title": "Nb consumer groups per state",
5849           "type": "timeseries"
5850         }
5851       ],
5852       "targets": [
5853         {
5854           "datasource": {
5855             "type": "datasource",
5856             "uid": "grafana"
5857           },
5858           "refId": "A"
5859         }
5860       ],
5861       "title": "Group Coordinator",
5862       "type": "row"
5863     },
5864     {
5865       "collapsed": true,
5866       "datasource": {
5867         "type": "datasource",
5868         "uid": "grafana"
5869       },
5870       "gridPos": {
5871         "h": 1,
5872         "w": 24,
5873         "x": 0,
5874         "y": 73
5875       },
5876       "id": 46,
5877       "panels": [
5878         {
5879           "datasource": {
5880             "type": "prometheus",
5881             "uid": "PBFA97CFB590B2093"
5882           },
5883           "description": "The number of messages produced converted to match the log.message.format.version.",
5884           "fieldConfig": {
5885             "defaults": {
5886               "color": {
5887                 "mode": "palette-classic"
5888               },
5889               "custom": {
5890                 "axisLabel": "",
5891                 "axisPlacement": "auto",
5892                 "barAlignment": 0,
5893                 "drawStyle": "line",
5894                 "fillOpacity": 10,
5895                 "gradientMode": "none",
5896                 "hideFrom": {
5897                   "legend": false,
5898                   "tooltip": false,
5899                   "viz": false
5900                 },
5901                 "lineInterpolation": "linear",
5902                 "lineWidth": 1,
5903                 "pointSize": 5,
5904                 "scaleDistribution": {
5905                   "type": "linear"
5906                 },
5907                 "showPoints": "never",
5908                 "spanNulls": true,
5909                 "stacking": {
5910                   "group": "A",
5911                   "mode": "none"
5912                 },
5913                 "thresholdsStyle": {
5914                   "mode": "off"
5915                 }
5916               },
5917               "links": [],
5918               "mappings": [],
5919               "min": 0,
5920               "thresholds": {
5921                 "mode": "absolute",
5922                 "steps": [
5923                   {
5924                     "color": "green"
5925                   },
5926                   {
5927                     "color": "red",
5928                     "value": 80
5929                   }
5930                 ]
5931               },
5932               "unit": "ops"
5933             },
5934             "overrides": []
5935           },
5936           "gridPos": {
5937             "h": 8,
5938             "w": 8,
5939             "x": 0,
5940             "y": 34
5941           },
5942           "id": 48,
5943           "options": {
5944             "legend": {
5945               "calcs": [
5946                 "mean",
5947                 "lastNotNull",
5948                 "max"
5949               ],
5950               "displayMode": "list",
5951               "placement": "bottom",
5952               "showLegend": true
5953             },
5954             "tooltip": {
5955               "mode": "single"
5956             }
5957           },
5958           "pluginVersion": "8.1.3",
5959           "targets": [
5960             {
5961               "datasource": {
5962                 "type": "prometheus",
5963                 "uid": "PBFA97CFB590B2093"
5964               },
5965               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5966               "hide": false,
5967               "interval": "",
5968               "legendFormat": "{{topic}}",
5969               "refId": "A"
5970             }
5971           ],
5972           "title": "Number of produced message conversion",
5973           "type": "timeseries"
5974         },
5975         {
5976           "datasource": {
5977             "type": "prometheus",
5978             "uid": "PBFA97CFB590B2093"
5979           },
5980           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5981           "fieldConfig": {
5982             "defaults": {
5983               "color": {
5984                 "mode": "palette-classic"
5985               },
5986               "custom": {
5987                 "axisLabel": "",
5988                 "axisPlacement": "auto",
5989                 "barAlignment": 0,
5990                 "drawStyle": "line",
5991                 "fillOpacity": 10,
5992                 "gradientMode": "none",
5993                 "hideFrom": {
5994                   "legend": false,
5995                   "tooltip": false,
5996                   "viz": false
5997                 },
5998                 "lineInterpolation": "linear",
5999                 "lineWidth": 1,
6000                 "pointSize": 5,
6001                 "scaleDistribution": {
6002                   "type": "linear"
6003                 },
6004                 "showPoints": "never",
6005                 "spanNulls": true,
6006                 "stacking": {
6007                   "group": "A",
6008                   "mode": "none"
6009                 },
6010                 "thresholdsStyle": {
6011                   "mode": "off"
6012                 }
6013               },
6014               "links": [],
6015               "mappings": [],
6016               "min": 0,
6017               "thresholds": {
6018                 "mode": "absolute",
6019                 "steps": [
6020                   {
6021                     "color": "green"
6022                   },
6023                   {
6024                     "color": "red",
6025                     "value": 80
6026                   }
6027                 ]
6028               },
6029               "unit": "ops"
6030             },
6031             "overrides": []
6032           },
6033           "gridPos": {
6034             "h": 8,
6035             "w": 8,
6036             "x": 8,
6037             "y": 34
6038           },
6039           "id": 51,
6040           "options": {
6041             "legend": {
6042               "calcs": [
6043                 "mean",
6044                 "lastNotNull",
6045                 "max"
6046               ],
6047               "displayMode": "list",
6048               "placement": "bottom",
6049               "showLegend": true
6050             },
6051             "tooltip": {
6052               "mode": "single"
6053             }
6054           },
6055           "pluginVersion": "8.1.3",
6056           "targets": [
6057             {
6058               "datasource": {
6059                 "type": "prometheus",
6060                 "uid": "PBFA97CFB590B2093"
6061               },
6062               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6063               "hide": false,
6064               "interval": "",
6065               "legendFormat": "{{topic}}",
6066               "refId": "A"
6067             }
6068           ],
6069           "title": "Number of consumed message conversion",
6070           "type": "timeseries"
6071         },
6072         {
6073           "datasource": {
6074             "type": "datasource",
6075             "uid": "grafana"
6076           },
6077           "description": "Number of connection per client version",
6078           "fieldConfig": {
6079             "defaults": {
6080               "color": {
6081                 "mode": "palette-classic"
6082               },
6083               "custom": {
6084                 "hideFrom": {
6085                   "legend": false,
6086                   "tooltip": false,
6087                   "viz": false
6088                 }
6089               },
6090               "decimals": 0,
6091               "mappings": [],
6092               "unit": "short"
6093             },
6094             "overrides": []
6095           },
6096           "gridPos": {
6097             "h": 8,
6098             "w": 8,
6099             "x": 16,
6100             "y": 34
6101           },
6102           "id": 96,
6103           "links": [],
6104           "options": {
6105             "legend": {
6106               "calcs": [],
6107               "displayMode": "table",
6108               "placement": "bottom",
6109               "showLegend": true,
6110               "values": [
6111                 "value"
6112               ]
6113             },
6114             "pieType": "pie",
6115             "reduceOptions": {
6116               "calcs": [
6117                 "lastNotNull"
6118               ],
6119               "fields": "",
6120               "values": false
6121             },
6122             "tooltip": {
6123               "mode": "single"
6124             }
6125           },
6126           "targets": [
6127             {
6128               "datasource": {
6129                 "type": "datasource",
6130                 "uid": "grafana"
6131               },
6132               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6133               "interval": "",
6134               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6135               "refId": "A"
6136             }
6137           ],
6138           "title": "Client version repartition",
6139           "type": "piechart"
6140         }
6141       ],
6142       "targets": [
6143         {
6144           "datasource": {
6145             "type": "datasource",
6146             "uid": "grafana"
6147           },
6148           "refId": "A"
6149         }
6150       ],
6151       "title": "Message Conversion",
6152       "type": "row"
6153     }
6154   ],
6155   "refresh": false,
6156   "schemaVersion": 38,
6157   "style": "dark",
6158   "tags": [],
6159   "templating": {
6160     "list": [
6161       {
6162         "current": {
6163           "selected": false,
6164           "text": "dev",
6165           "value": "dev"
6166         },
6167         "datasource": {
6168           "type": "prometheus",
6169           "uid": "PBFA97CFB590B2093"
6170         },
6171         "definition": "label_values(env)",
6172         "hide": 0,
6173         "includeAll": false,
6174         "label": "Environment",
6175         "multi": false,
6176         "name": "env",
6177         "options": [],
6178         "query": {
6179           "query": "label_values(env)",
6180           "refId": "Prometheus-env-Variable-Query"
6181         },
6182         "refresh": 1,
6183         "regex": "",
6184         "skipUrlSync": false,
6185         "sort": 0,
6186         "tagValuesQuery": "",
6187         "tagsQuery": "",
6188         "type": "query",
6189         "useTags": false
6190       },
6191       {
6192         "current": {
6193           "selected": true,
6194           "text": [
6195             "All"
6196           ],
6197           "value": [
6198             "$__all"
6199           ]
6200         },
6201         "datasource": {
6202           "type": "prometheus",
6203           "uid": "PBFA97CFB590B2093"
6204         },
6205         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6206         "hide": 0,
6207         "includeAll": true,
6208         "label": "Instance",
6209         "multi": true,
6210         "name": "instance",
6211         "options": [],
6212         "query": {
6213           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6214           "refId": "Prometheus-instance-Variable-Query"
6215         },
6216         "refresh": 1,
6217         "regex": "",
6218         "skipUrlSync": false,
6219         "sort": 0,
6220         "tagValuesQuery": "",
6221         "tagsQuery": "",
6222         "type": "query",
6223         "useTags": false
6224       },
6225       {
6226         "allValue": "",
6227         "current": {
6228           "selected": true,
6229           "text": [
6230             "0.95"
6231           ],
6232           "value": [
6233             "0.95"
6234           ]
6235         },
6236         "datasource": {
6237           "type": "prometheus",
6238           "uid": "PBFA97CFB590B2093"
6239         },
6240         "definition": "label_values(quantile)",
6241         "hide": 0,
6242         "includeAll": true,
6243         "label": "Percentile",
6244         "multi": true,
6245         "name": "percentile",
6246         "options": [],
6247         "query": {
6248           "query": "label_values(quantile)",
6249           "refId": "Prometheus-percentile-Variable-Query"
6250         },
6251         "refresh": 1,
6252         "regex": "",
6253         "skipUrlSync": false,
6254         "sort": 0,
6255         "tagValuesQuery": "",
6256         "tagsQuery": "",
6257         "type": "query",
6258         "useTags": false
6259       }
6260     ]
6261   },
6262   "time": {
6263     "from": "2023-08-08T18:45:09.420Z",
6264     "to": "2023-08-08T19:25:09.420Z"
6265   },
6266   "timepicker": {
6267     "refresh_intervals": [
6268       "5s",
6269       "10s",
6270       "30s",
6271       "1m",
6272       "5m",
6273       "15m",
6274       "30m",
6275       "1h",
6276       "2h",
6277       "1d"
6278     ],
6279     "time_options": [
6280       "5m",
6281       "15m",
6282       "1h",
6283       "6h",
6284       "12h",
6285       "24h",
6286       "2d",
6287       "7d",
6288       "30d"
6289     ]
6290   },
6291   "timezone": "browser",
6292   "title": "Kafka cluster",
6293   "uid": "qu-QZdfZz",
6294   "version": 1,
6295   "weekStart": ""
6296 }