juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of active controllers in the cluster.",
  64       "fieldConfig": {
  65         "defaults": {
  66           "mappings": [
  67             {
  68               "options": {
  69                 "match": "null",
  70                 "result": {
  71                   "text": "N/A"
  72                 }
  73               },
  74               "type": "special"
  75             }
  76           ],
  77           "thresholds": {
  78             "mode": "absolute",
  79             "steps": [
  80               {
  81                 "color": "#299c46",
  82                 "value": null
  83               },
  84               {
  85                 "color": "#e5ac0e",
  86                 "value": 2
  87               },
  88               {
  89                 "color": "#bf1b00"
  90               }
  91             ]
  92           },
  93           "unit": "none"
  94         },
  95         "overrides": []
  96       },
  97       "gridPos": {
  98         "h": 4,
  99         "w": 4,
 100         "x": 0,
 101         "y": 1
 102       },
 103       "id": 12,
 104       "links": [],
 105       "maxDataPoints": 100,
 106       "options": {
 107         "colorMode": "value",
 108         "graphMode": "none",
 109         "justifyMode": "auto",
 110         "orientation": "vertical",
 111         "reduceOptions": {
 112           "calcs": [
 113             "lastNotNull"
 114           ],
 115           "fields": "",
 116           "values": false
 117         },
 118         "text": {},
 119         "textMode": "value_and_name"
 120       },
 121       "pluginVersion": "10.0.3",
 122       "targets": [
 123         {
 124           "datasource": {
 125             "type": "prometheus",
 126             "uid": "PBFA97CFB590B2093"
 127           },
 128           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 129           "format": "time_series",
 130           "instant": true,
 131           "interval": "",
 132           "intervalFactor": 1,
 133           "legendFormat": "{{instance}}",
 134           "refId": "A"
 135         }
 136       ],
 137       "title": "Active Controllers",
 138       "type": "stat"
 139     },
 140     {
 141       "datasource": {
 142         "type": "prometheus",
 143         "uid": "PBFA97CFB590B2093"
 144       },
 145       "description": "Number of partitions under min insync replicas.",
 146       "fieldConfig": {
 147         "defaults": {
 148           "color": {
 149             "mode": "thresholds"
 150           },
 151           "mappings": [
 152             {
 153               "options": {
 154                 "match": "null",
 155                 "result": {
 156                   "text": "N/A"
 157                 }
 158               },
 159               "type": "special"
 160             }
 161           ],
 162           "thresholds": {
 163             "mode": "absolute",
 164             "steps": [
 165               {
 166                 "color": "#508642",
 167                 "value": null
 168               },
 169               {
 170                 "color": "rgba(237, 129, 40, 0.89)",
 171                 "value": 1
 172               },
 173               {
 174                 "color": "#bf1b00",
 175                 "value": 5
 176               }
 177             ]
 178           },
 179           "unit": "none"
 180         },
 181         "overrides": []
 182       },
 183       "gridPos": {
 184         "h": 4,
 185         "w": 4,
 186         "x": 4,
 187         "y": 1
 188       },
 189       "id": 32,
 190       "links": [],
 191       "maxDataPoints": 100,
 192       "options": {
 193         "colorMode": "value",
 194         "graphMode": "none",
 195         "justifyMode": "auto",
 196         "orientation": "horizontal",
 197         "reduceOptions": {
 198           "calcs": [
 199             "lastNotNull"
 200           ],
 201           "fields": "",
 202           "values": false
 203         },
 204         "text": {},
 205         "textMode": "auto"
 206       },
 207       "pluginVersion": "10.0.3",
 208       "targets": [
 209         {
 210           "datasource": {
 211             "type": "prometheus",
 212             "uid": "PBFA97CFB590B2093"
 213           },
 214           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 215           "format": "time_series",
 216           "hide": false,
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 2,
 220           "legendFormat": "",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Under Min ISR Partitions",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#508642",
 254                 "value": null
 255               },
 256               {
 257                 "color": "#ef843c",
 258                 "value": 1
 259               },
 260               {
 261                 "color": "#bf1b00",
 262                 "value": 1
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 4,
 273         "x": 8,
 274         "y": 1
 275       },
 276       "id": 22,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Offline Partitions Count",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 12,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#508642",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 1
 430               },
 431               {
 432                 "color": "#bf1b00",
 433                 "value": 5
 434               }
 435             ]
 436           },
 437           "unit": "none"
 438         },
 439         "overrides": []
 440       },
 441       "gridPos": {
 442         "h": 4,
 443         "w": 4,
 444         "x": 0,
 445         "y": 5
 446       },
 447       "id": 20,
 448       "links": [],
 449       "maxDataPoints": 100,
 450       "options": {
 451         "colorMode": "value",
 452         "graphMode": "none",
 453         "justifyMode": "auto",
 454         "orientation": "horizontal",
 455         "reduceOptions": {
 456           "calcs": [
 457             "lastNotNull"
 458           ],
 459           "fields": "",
 460           "values": false
 461         },
 462         "text": {},
 463         "textMode": "auto"
 464       },
 465       "pluginVersion": "10.0.3",
 466       "targets": [
 467         {
 468           "datasource": {
 469             "type": "prometheus",
 470             "uid": "PBFA97CFB590B2093"
 471           },
 472           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 473           "format": "time_series",
 474           "hide": false,
 475           "instant": true,
 476           "interval": "",
 477           "intervalFactor": 2,
 478           "legendFormat": "",
 479           "refId": "A"
 480         }
 481       ],
 482       "title": "Under Replicated Partitions",
 483       "type": "stat"
 484     },
 485     {
 486       "datasource": {
 487         "type": "prometheus",
 488         "uid": "PBFA97CFB590B2093"
 489       },
 490       "description": "Number of Brokers Online",
 491       "fieldConfig": {
 492         "defaults": {
 493           "color": {
 494             "mode": "thresholds"
 495           },
 496           "mappings": [
 497             {
 498               "options": {
 499                 "match": "null",
 500                 "result": {
 501                   "text": "N/A"
 502                 }
 503               },
 504               "type": "special"
 505             }
 506           ],
 507           "thresholds": {
 508             "mode": "absolute",
 509             "steps": [
 510               {
 511                 "color": "#d44a3a",
 512                 "value": null
 513               },
 514               {
 515                 "color": "rgba(237, 129, 40, 0.89)",
 516                 "value": 0
 517               },
 518               {
 519                 "color": "#299c46",
 520                 "value": 2
 521               }
 522             ]
 523           },
 524           "unit": "none"
 525         },
 526         "overrides": []
 527       },
 528       "gridPos": {
 529         "h": 4,
 530         "w": 4,
 531         "x": 4,
 532         "y": 5
 533       },
 534       "id": 14,
 535       "links": [],
 536       "maxDataPoints": 100,
 537       "options": {
 538         "colorMode": "value",
 539         "graphMode": "none",
 540         "justifyMode": "auto",
 541         "orientation": "horizontal",
 542         "reduceOptions": {
 543           "calcs": [
 544             "lastNotNull"
 545           ],
 546           "fields": "",
 547           "values": false
 548         },
 549         "text": {},
 550         "textMode": "auto"
 551       },
 552       "pluginVersion": "10.0.3",
 553       "repeatDirection": "h",
 554       "targets": [
 555         {
 556           "datasource": {
 557             "type": "prometheus",
 558             "uid": "PBFA97CFB590B2093"
 559           },
 560           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 561           "format": "time_series",
 562           "instant": true,
 563           "interval": "",
 564           "intervalFactor": 1,
 565           "legendFormat": "",
 566           "refId": "A"
 567         }
 568       ],
 569       "title": "Brokers Online",
 570       "type": "stat"
 571     },
 572     {
 573       "datasource": {
 574         "type": "prometheus",
 575         "uid": "PBFA97CFB590B2093"
 576       },
 577       "description": "Partitions that are online",
 578       "fieldConfig": {
 579         "defaults": {
 580           "color": {
 581             "mode": "thresholds"
 582           },
 583           "mappings": [
 584             {
 585               "options": {
 586                 "match": "null",
 587                 "result": {
 588                   "text": "N/A"
 589                 }
 590               },
 591               "type": "special"
 592             }
 593           ],
 594           "thresholds": {
 595             "mode": "absolute",
 596             "steps": [
 597               {
 598                 "color": "#d44a3a",
 599                 "value": null
 600               },
 601               {
 602                 "color": "rgba(237, 129, 40, 0.89)",
 603                 "value": 0
 604               },
 605               {
 606                 "color": "#299c46",
 607                 "value": 0
 608               }
 609             ]
 610           },
 611           "unit": "none"
 612         },
 613         "overrides": []
 614       },
 615       "gridPos": {
 616         "h": 4,
 617         "w": 4,
 618         "x": 8,
 619         "y": 5
 620       },
 621       "id": 18,
 622       "links": [],
 623       "maxDataPoints": 100,
 624       "options": {
 625         "colorMode": "value",
 626         "graphMode": "none",
 627         "justifyMode": "auto",
 628         "orientation": "horizontal",
 629         "reduceOptions": {
 630           "calcs": [
 631             "lastNotNull"
 632           ],
 633           "fields": "",
 634           "values": false
 635         },
 636         "text": {},
 637         "textMode": "auto"
 638       },
 639       "pluginVersion": "10.0.3",
 640       "targets": [
 641         {
 642           "datasource": {
 643             "type": "prometheus",
 644             "uid": "PBFA97CFB590B2093"
 645           },
 646           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 647           "format": "time_series",
 648           "instant": true,
 649           "interval": "",
 650           "intervalFactor": 1,
 651           "legendFormat": "",
 652           "refId": "A"
 653         }
 654       ],
 655       "title": "Online Partitions",
 656       "type": "stat"
 657     },
 658     {
 659       "datasource": {
 660         "type": "prometheus",
 661         "uid": "PBFA97CFB590B2093"
 662       },
 663       "description": "Unclean leader election rate",
 664       "fieldConfig": {
 665         "defaults": {
 666           "color": {
 667             "mode": "thresholds"
 668           },
 669           "mappings": [
 670             {
 671               "options": {
 672                 "match": "null",
 673                 "result": {
 674                   "text": "N/A"
 675                 }
 676               },
 677               "type": "special"
 678             }
 679           ],
 680           "thresholds": {
 681             "mode": "absolute",
 682             "steps": [
 683               {
 684                 "color": "#299c46",
 685                 "value": null
 686               },
 687               {
 688                 "color": "rgba(237, 129, 40, 0.89)",
 689                 "value": 2
 690               },
 691               {
 692                 "color": "#d44a3a"
 693               }
 694             ]
 695           },
 696           "unit": "none"
 697         },
 698         "overrides": []
 699       },
 700       "gridPos": {
 701         "h": 4,
 702         "w": 4,
 703         "x": 12,
 704         "y": 5
 705       },
 706       "id": 16,
 707       "links": [],
 708       "maxDataPoints": 100,
 709       "options": {
 710         "colorMode": "value",
 711         "graphMode": "none",
 712         "justifyMode": "auto",
 713         "orientation": "horizontal",
 714         "reduceOptions": {
 715           "calcs": [
 716             "lastNotNull"
 717           ],
 718           "fields": "",
 719           "values": false
 720         },
 721         "text": {},
 722         "textMode": "auto"
 723       },
 724       "pluginVersion": "10.0.3",
 725       "targets": [
 726         {
 727           "datasource": {
 728             "type": "prometheus",
 729             "uid": "PBFA97CFB590B2093"
 730           },
 731           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 732           "format": "time_series",
 733           "instant": true,
 734           "interval": "",
 735           "intervalFactor": 1,
 736           "legendFormat": "",
 737           "refId": "A"
 738         }
 739       ],
 740       "title": "Unclean Leader Election Rate",
 741       "type": "stat"
 742     },
 743     {
 744       "datasource": {
 745         "type": "prometheus",
 746         "uid": "PBFA97CFB590B2093"
 747       },
 748       "fieldConfig": {
 749         "defaults": {
 750           "color": {
 751             "mode": "palette-classic"
 752           },
 753           "custom": {
 754             "axisCenteredZero": false,
 755             "axisColorMode": "text",
 756             "axisLabel": "Bytes/s",
 757             "axisPlacement": "auto",
 758             "barAlignment": 0,
 759             "drawStyle": "line",
 760             "fillOpacity": 10,
 761             "gradientMode": "none",
 762             "hideFrom": {
 763               "legend": false,
 764               "tooltip": false,
 765               "viz": false
 766             },
 767             "lineInterpolation": "linear",
 768             "lineWidth": 2,
 769             "pointSize": 5,
 770             "scaleDistribution": {
 771               "type": "linear"
 772             },
 773             "showPoints": "never",
 774             "spanNulls": false,
 775             "stacking": {
 776               "group": "A",
 777               "mode": "none"
 778             },
 779             "thresholdsStyle": {
 780               "mode": "off"
 781             }
 782           },
 783           "links": [],
 784           "mappings": [],
 785           "thresholds": {
 786             "mode": "absolute",
 787             "steps": [
 788               {
 789                 "color": "green",
 790                 "value": null
 791               },
 792               {
 793                 "color": "red",
 794                 "value": 80
 795               }
 796             ]
 797           },
 798           "unit": "Bps"
 799         },
 800         "overrides": []
 801       },
 802       "gridPos": {
 803         "h": 8,
 804         "w": 8,
 805         "x": 0,
 806         "y": 9
 807       },
 808       "id": 84,
 809       "links": [],
 810       "options": {
 811         "legend": {
 812           "calcs": [
 813             "mean",
 814             "lastNotNull",
 815             "max"
 816           ],
 817           "displayMode": "list",
 818           "placement": "bottom",
 819           "showLegend": true
 820         },
 821         "tooltip": {
 822           "mode": "single",
 823           "sort": "none"
 824         }
 825       },
 826       "pluginVersion": "8.1.3",
 827       "targets": [
 828         {
 829           "datasource": {
 830             "type": "prometheus",
 831             "uid": "PBFA97CFB590B2093"
 832           },
 833           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 834           "format": "time_series",
 835           "interval": "",
 836           "intervalFactor": 2,
 837           "legendFormat": "Bytes in",
 838           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 839           "refId": "A",
 840           "step": 4
 841         },
 842         {
 843           "datasource": {
 844             "type": "prometheus",
 845             "uid": "PBFA97CFB590B2093"
 846           },
 847           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 848           "format": "time_series",
 849           "hide": false,
 850           "interval": "",
 851           "intervalFactor": 2,
 852           "legendFormat": "Bytes out",
 853           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 854           "refId": "B",
 855           "step": 4
 856         }
 857       ],
 858       "title": "Broker network throughput",
 859       "type": "timeseries"
 860     },
 861     {
 862       "datasource": {
 863         "type": "prometheus",
 864         "uid": "PBFA97CFB590B2093"
 865       },
 866       "description": "",
 867       "fieldConfig": {
 868         "defaults": {
 869           "color": {
 870             "mode": "palette-classic"
 871           },
 872           "custom": {
 873             "axisCenteredZero": false,
 874             "axisColorMode": "text",
 875             "axisLabel": "",
 876             "axisPlacement": "auto",
 877             "barAlignment": 0,
 878             "drawStyle": "line",
 879             "fillOpacity": 10,
 880             "gradientMode": "none",
 881             "hideFrom": {
 882               "legend": false,
 883               "tooltip": false,
 884               "viz": false
 885             },
 886             "lineInterpolation": "linear",
 887             "lineWidth": 1,
 888             "pointSize": 5,
 889             "scaleDistribution": {
 890               "type": "linear"
 891             },
 892             "showPoints": "never",
 893             "spanNulls": true,
 894             "stacking": {
 895               "group": "A",
 896               "mode": "none"
 897             },
 898             "thresholdsStyle": {
 899               "mode": "off"
 900             }
 901           },
 902           "decimals": 0,
 903           "links": [],
 904           "mappings": [],
 905           "min": 0,
 906           "thresholds": {
 907             "mode": "absolute",
 908             "steps": [
 909               {
 910                 "color": "green"
 911               },
 912               {
 913                 "color": "red",
 914                 "value": 80
 915               }
 916             ]
 917           },
 918           "unit": "partitions"
 919         },
 920         "overrides": [
 921           {
 922             "__systemRef": "hideSeriesFrom",
 923             "matcher": {
 924               "id": "byNames",
 925               "options": {
 926                 "mode": "exclude",
 927                 "names": [
 928                   "kafka-3:80"
 929                 ],
 930                 "prefix": "All except:",
 931                 "readOnly": true
 932               }
 933             },
 934             "properties": [
 935               {
 936                 "id": "custom.hideFrom",
 937                 "value": {
 938                   "legend": false,
 939                   "tooltip": false,
 940                   "viz": true
 941                 }
 942               }
 943             ]
 944           }
 945         ]
 946       },
 947       "gridPos": {
 948         "h": 8,
 949         "w": 16,
 950         "x": 8,
 951         "y": 9
 952       },
 953       "id": 123,
 954       "options": {
 955         "legend": {
 956           "calcs": [
 957             "lastNotNull",
 958             "max"
 959           ],
 960           "displayMode": "table",
 961           "placement": "bottom",
 962           "showLegend": true
 963         },
 964         "tooltip": {
 965           "mode": "single",
 966           "sort": "none"
 967         }
 968       },
 969       "pluginVersion": "8.1.3",
 970       "targets": [
 971         {
 972           "datasource": {
 973             "type": "prometheus",
 974             "uid": "PBFA97CFB590B2093"
 975           },
 976           "editorMode": "code",
 977           "exemplar": false,
 978           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 979           "hide": false,
 980           "interval": "",
 981           "legendFormat": "{{instance}}",
 982           "range": true,
 983           "refId": "A"
 984         }
 985       ],
 986       "title": "Underreplicated Partitions",
 987       "type": "timeseries"
 988     },
 989     {
 990       "collapsed": false,
 991       "datasource": {
 992         "type": "datasource",
 993         "uid": "grafana"
 994       },
 995       "gridPos": {
 996         "h": 1,
 997         "w": 24,
 998         "x": 0,
 999         "y": 17
1000       },
1001       "id": 31,
1002       "panels": [],
1003       "targets": [
1004         {
1005           "datasource": {
1006             "type": "datasource",
1007             "uid": "grafana"
1008           },
1009           "refId": "A"
1010         }
1011       ],
1012       "title": "Request rate",
1013       "type": "row"
1014     },
1015     {
1016       "datasource": {
1017         "type": "prometheus",
1018         "uid": "PBFA97CFB590B2093"
1019       },
1020       "description": "Produce request rate.",
1021       "fieldConfig": {
1022         "defaults": {
1023           "mappings": [
1024             {
1025               "options": {
1026                 "match": "null",
1027                 "result": {
1028                   "text": "N/A"
1029                 }
1030               },
1031               "type": "special"
1032             }
1033           ],
1034           "thresholds": {
1035             "mode": "absolute",
1036             "steps": [
1037               {
1038                 "color": "green",
1039                 "value": null
1040               }
1041             ]
1042           },
1043           "unit": "none"
1044         },
1045         "overrides": []
1046       },
1047       "gridPos": {
1048         "h": 4,
1049         "w": 4,
1050         "x": 0,
1051         "y": 18
1052       },
1053       "id": 93,
1054       "links": [],
1055       "maxDataPoints": 100,
1056       "options": {
1057         "colorMode": "value",
1058         "graphMode": "area",
1059         "justifyMode": "auto",
1060         "orientation": "horizontal",
1061         "reduceOptions": {
1062           "calcs": [
1063             "lastNotNull"
1064           ],
1065           "fields": "",
1066           "values": false
1067         },
1068         "text": {},
1069         "textMode": "auto"
1070       },
1071       "pluginVersion": "10.0.3",
1072       "targets": [
1073         {
1074           "datasource": {
1075             "type": "prometheus",
1076             "uid": "PBFA97CFB590B2093"
1077           },
1078           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1079           "interval": "",
1080           "legendFormat": "",
1081           "refId": "A"
1082         }
1083       ],
1084       "title": "All Request Per Sec",
1085       "type": "stat"
1086     },
1087     {
1088       "datasource": {
1089         "type": "prometheus",
1090         "uid": "PBFA97CFB590B2093"
1091       },
1092       "description": "Produce request rate.",
1093       "fieldConfig": {
1094         "defaults": {
1095           "mappings": [
1096             {
1097               "options": {
1098                 "match": "null",
1099                 "result": {
1100                   "text": "N/A"
1101                 }
1102               },
1103               "type": "special"
1104             }
1105           ],
1106           "thresholds": {
1107             "mode": "absolute",
1108             "steps": [
1109               {
1110                 "color": "green",
1111                 "value": null
1112               }
1113             ]
1114           },
1115           "unit": "none"
1116         },
1117         "overrides": []
1118       },
1119       "gridPos": {
1120         "h": 4,
1121         "w": 4,
1122         "x": 4,
1123         "y": 18
1124       },
1125       "id": 35,
1126       "links": [],
1127       "maxDataPoints": 100,
1128       "options": {
1129         "colorMode": "value",
1130         "graphMode": "area",
1131         "justifyMode": "auto",
1132         "orientation": "horizontal",
1133         "reduceOptions": {
1134           "calcs": [
1135             "lastNotNull"
1136           ],
1137           "fields": "",
1138           "values": false
1139         },
1140         "text": {},
1141         "textMode": "auto"
1142       },
1143       "pluginVersion": "10.0.3",
1144       "targets": [
1145         {
1146           "datasource": {
1147             "type": "prometheus",
1148             "uid": "PBFA97CFB590B2093"
1149           },
1150           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1151           "interval": "",
1152           "legendFormat": "",
1153           "refId": "A"
1154         }
1155       ],
1156       "title": "Produce Request Per Sec",
1157       "type": "stat"
1158     },
1159     {
1160       "datasource": {
1161         "type": "prometheus",
1162         "uid": "PBFA97CFB590B2093"
1163       },
1164       "description": "Fetch request rate.",
1165       "fieldConfig": {
1166         "defaults": {
1167           "mappings": [
1168             {
1169               "options": {
1170                 "match": "null",
1171                 "result": {
1172                   "text": "N/A"
1173                 }
1174               },
1175               "type": "special"
1176             }
1177           ],
1178           "thresholds": {
1179             "mode": "absolute",
1180             "steps": [
1181               {
1182                 "color": "green",
1183                 "value": null
1184               }
1185             ]
1186           },
1187           "unit": "none"
1188         },
1189         "overrides": []
1190       },
1191       "gridPos": {
1192         "h": 4,
1193         "w": 4,
1194         "x": 8,
1195         "y": 18
1196       },
1197       "id": 37,
1198       "links": [],
1199       "maxDataPoints": 100,
1200       "options": {
1201         "colorMode": "value",
1202         "graphMode": "area",
1203         "justifyMode": "auto",
1204         "orientation": "horizontal",
1205         "reduceOptions": {
1206           "calcs": [
1207             "lastNotNull"
1208           ],
1209           "fields": "",
1210           "values": false
1211         },
1212         "text": {},
1213         "textMode": "auto"
1214       },
1215       "pluginVersion": "10.0.3",
1216       "targets": [
1217         {
1218           "datasource": {
1219             "type": "prometheus",
1220             "uid": "PBFA97CFB590B2093"
1221           },
1222           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1223           "interval": "",
1224           "legendFormat": "",
1225           "refId": "A"
1226         }
1227       ],
1228       "title": "Consumer Fetch Request Per Sec",
1229       "type": "stat"
1230     },
1231     {
1232       "datasource": {
1233         "type": "prometheus",
1234         "uid": "PBFA97CFB590B2093"
1235       },
1236       "description": "Fetch request rate.",
1237       "fieldConfig": {
1238         "defaults": {
1239           "mappings": [
1240             {
1241               "options": {
1242                 "match": "null",
1243                 "result": {
1244                   "text": "N/A"
1245                 }
1246               },
1247               "type": "special"
1248             }
1249           ],
1250           "thresholds": {
1251             "mode": "absolute",
1252             "steps": [
1253               {
1254                 "color": "green",
1255                 "value": null
1256               }
1257             ]
1258           },
1259           "unit": "none"
1260         },
1261         "overrides": []
1262       },
1263       "gridPos": {
1264         "h": 4,
1265         "w": 4,
1266         "x": 0,
1267         "y": 22
1268       },
1269       "id": 94,
1270       "links": [],
1271       "maxDataPoints": 100,
1272       "options": {
1273         "colorMode": "value",
1274         "graphMode": "area",
1275         "justifyMode": "auto",
1276         "orientation": "horizontal",
1277         "reduceOptions": {
1278           "calcs": [
1279             "lastNotNull"
1280           ],
1281           "fields": "",
1282           "values": false
1283         },
1284         "text": {},
1285         "textMode": "auto"
1286       },
1287       "pluginVersion": "10.0.3",
1288       "targets": [
1289         {
1290           "datasource": {
1291             "type": "prometheus",
1292             "uid": "PBFA97CFB590B2093"
1293           },
1294           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1295           "interval": "",
1296           "legendFormat": "",
1297           "refId": "A"
1298         }
1299       ],
1300       "title": "Broker Fetch Request Per Sec",
1301       "type": "stat"
1302     },
1303     {
1304       "datasource": {
1305         "type": "prometheus",
1306         "uid": "PBFA97CFB590B2093"
1307       },
1308       "description": "Offset Commit request rate.",
1309       "fieldConfig": {
1310         "defaults": {
1311           "mappings": [
1312             {
1313               "options": {
1314                 "match": "null",
1315                 "result": {
1316                   "text": "N/A"
1317                 }
1318               },
1319               "type": "special"
1320             }
1321           ],
1322           "thresholds": {
1323             "mode": "absolute",
1324             "steps": [
1325               {
1326                 "color": "green",
1327                 "value": null
1328               }
1329             ]
1330           },
1331           "unit": "none"
1332         },
1333         "overrides": []
1334       },
1335       "gridPos": {
1336         "h": 4,
1337         "w": 4,
1338         "x": 4,
1339         "y": 22
1340       },
1341       "id": 38,
1342       "links": [],
1343       "maxDataPoints": 100,
1344       "options": {
1345         "colorMode": "value",
1346         "graphMode": "area",
1347         "justifyMode": "auto",
1348         "orientation": "horizontal",
1349         "reduceOptions": {
1350           "calcs": [
1351             "lastNotNull"
1352           ],
1353           "fields": "",
1354           "values": false
1355         },
1356         "text": {},
1357         "textMode": "auto"
1358       },
1359       "pluginVersion": "10.0.3",
1360       "targets": [
1361         {
1362           "datasource": {
1363             "type": "prometheus",
1364             "uid": "PBFA97CFB590B2093"
1365           },
1366           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1367           "interval": "",
1368           "legendFormat": "",
1369           "refId": "A"
1370         }
1371       ],
1372       "title": "Offset Commit Request Per Sec",
1373       "type": "stat"
1374     },
1375     {
1376       "datasource": {
1377         "type": "prometheus",
1378         "uid": "PBFA97CFB590B2093"
1379       },
1380       "description": "Metadata request rate.",
1381       "fieldConfig": {
1382         "defaults": {
1383           "mappings": [
1384             {
1385               "options": {
1386                 "match": "null",
1387                 "result": {
1388                   "text": "N/A"
1389                 }
1390               },
1391               "type": "special"
1392             }
1393           ],
1394           "thresholds": {
1395             "mode": "absolute",
1396             "steps": [
1397               {
1398                 "color": "green",
1399                 "value": null
1400               }
1401             ]
1402           },
1403           "unit": "none"
1404         },
1405         "overrides": []
1406       },
1407       "gridPos": {
1408         "h": 4,
1409         "w": 4,
1410         "x": 8,
1411         "y": 22
1412       },
1413       "id": 36,
1414       "links": [],
1415       "maxDataPoints": 100,
1416       "options": {
1417         "colorMode": "value",
1418         "graphMode": "area",
1419         "justifyMode": "auto",
1420         "orientation": "horizontal",
1421         "reduceOptions": {
1422           "calcs": [
1423             "lastNotNull"
1424           ],
1425           "fields": "",
1426           "values": false
1427         },
1428         "text": {},
1429         "textMode": "auto"
1430       },
1431       "pluginVersion": "10.0.3",
1432       "targets": [
1433         {
1434           "datasource": {
1435             "type": "prometheus",
1436             "uid": "PBFA97CFB590B2093"
1437           },
1438           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1439           "interval": "",
1440           "legendFormat": "",
1441           "refId": "A"
1442         }
1443       ],
1444       "title": "Metadata Request Per Sec",
1445       "type": "stat"
1446     },
1447     {
1448       "collapsed": false,
1449       "datasource": {
1450         "type": "datasource",
1451         "uid": "grafana"
1452       },
1453       "gridPos": {
1454         "h": 1,
1455         "w": 24,
1456         "x": 0,
1457         "y": 26
1458       },
1459       "id": 40,
1460       "panels": [],
1461       "targets": [
1462         {
1463           "datasource": {
1464             "type": "datasource",
1465             "uid": "grafana"
1466           },
1467           "refId": "A"
1468         }
1469       ],
1470       "title": "System",
1471       "type": "row"
1472     },
1473     {
1474       "datasource": {
1475         "type": "prometheus",
1476         "uid": "PBFA97CFB590B2093"
1477       },
1478       "fieldConfig": {
1479         "defaults": {
1480           "color": {
1481             "mode": "palette-classic"
1482           },
1483           "custom": {
1484             "axisCenteredZero": false,
1485             "axisColorMode": "text",
1486             "axisLabel": "Cores",
1487             "axisPlacement": "auto",
1488             "barAlignment": 0,
1489             "drawStyle": "line",
1490             "fillOpacity": 10,
1491             "gradientMode": "none",
1492             "hideFrom": {
1493               "legend": false,
1494               "tooltip": false,
1495               "viz": false
1496             },
1497             "lineInterpolation": "linear",
1498             "lineWidth": 2,
1499             "pointSize": 5,
1500             "scaleDistribution": {
1501               "type": "linear"
1502             },
1503             "showPoints": "never",
1504             "spanNulls": false,
1505             "stacking": {
1506               "group": "A",
1507               "mode": "none"
1508             },
1509             "thresholdsStyle": {
1510               "mode": "off"
1511             }
1512           },
1513           "links": [],
1514           "mappings": [],
1515           "thresholds": {
1516             "mode": "absolute",
1517             "steps": [
1518               {
1519                 "color": "green",
1520                 "value": null
1521               },
1522               {
1523                 "color": "red",
1524                 "value": 80
1525               }
1526             ]
1527           },
1528           "unit": "percent"
1529         },
1530         "overrides": [
1531           {
1532             "matcher": {
1533               "id": "byName",
1534               "options": "localhost:7071"
1535             },
1536             "properties": [
1537               {
1538                 "id": "color",
1539                 "value": {
1540                   "fixedColor": "#629E51",
1541                   "mode": "fixed"
1542                 }
1543               }
1544             ]
1545           }
1546         ]
1547       },
1548       "gridPos": {
1549         "h": 8,
1550         "w": 8,
1551         "x": 0,
1552         "y": 27
1553       },
1554       "id": 27,
1555       "links": [],
1556       "options": {
1557         "legend": {
1558           "calcs": [
1559             "mean",
1560             "lastNotNull",
1561             "max"
1562           ],
1563           "displayMode": "table",
1564           "placement": "bottom",
1565           "showLegend": true
1566         },
1567         "tooltip": {
1568           "mode": "single",
1569           "sort": "none"
1570         }
1571       },
1572       "pluginVersion": "8.1.3",
1573       "targets": [
1574         {
1575           "datasource": {
1576             "type": "prometheus",
1577             "uid": "PBFA97CFB590B2093"
1578           },
1579           "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1580           "format": "time_series",
1581           "interval": "",
1582           "intervalFactor": 2,
1583           "legendFormat": "{{instance}}",
1584           "metric": "process_cpu_secondspersec",
1585           "refId": "A",
1586           "step": 4
1587         }
1588       ],
1589       "title": "CPU Usage",
1590       "type": "timeseries"
1591     },
1592     {
1593       "datasource": {
1594         "type": "prometheus",
1595         "uid": "PBFA97CFB590B2093"
1596       },
1597       "fieldConfig": {
1598         "defaults": {
1599           "color": {
1600             "mode": "palette-classic"
1601           },
1602           "custom": {
1603             "axisCenteredZero": false,
1604             "axisColorMode": "text",
1605             "axisLabel": "Memory",
1606             "axisPlacement": "auto",
1607             "barAlignment": 0,
1608             "drawStyle": "line",
1609             "fillOpacity": 10,
1610             "gradientMode": "none",
1611             "hideFrom": {
1612               "legend": false,
1613               "tooltip": false,
1614               "viz": false
1615             },
1616             "lineInterpolation": "linear",
1617             "lineWidth": 2,
1618             "pointSize": 5,
1619             "scaleDistribution": {
1620               "type": "linear"
1621             },
1622             "showPoints": "never",
1623             "spanNulls": false,
1624             "stacking": {
1625               "group": "A",
1626               "mode": "none"
1627             },
1628             "thresholdsStyle": {
1629               "mode": "off"
1630             }
1631           },
1632           "links": [],
1633           "mappings": [],
1634           "thresholds": {
1635             "mode": "absolute",
1636             "steps": [
1637               {
1638                 "color": "green",
1639                 "value": null
1640               },
1641               {
1642                 "color": "red",
1643                 "value": 80
1644               }
1645             ]
1646           },
1647           "unit": "bytes"
1648         },
1649         "overrides": [
1650           {
1651             "matcher": {
1652               "id": "byName",
1653               "options": "localhost:7071"
1654             },
1655             "properties": [
1656               {
1657                 "id": "color",
1658                 "value": {
1659                   "fixedColor": "#BA43A9",
1660                   "mode": "fixed"
1661                 }
1662               }
1663             ]
1664           }
1665         ]
1666       },
1667       "gridPos": {
1668         "h": 8,
1669         "w": 8,
1670         "x": 8,
1671         "y": 27
1672       },
1673       "id": 2,
1674       "links": [],
1675       "options": {
1676         "legend": {
1677           "calcs": [
1678             "mean",
1679             "lastNotNull",
1680             "max"
1681           ],
1682           "displayMode": "table",
1683           "placement": "bottom",
1684           "showLegend": true
1685         },
1686         "tooltip": {
1687           "mode": "single",
1688           "sort": "none"
1689         }
1690       },
1691       "pluginVersion": "8.1.3",
1692       "targets": [
1693         {
1694           "datasource": {
1695             "type": "prometheus",
1696             "uid": "PBFA97CFB590B2093"
1697           },
1698           "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1699           "interval": "",
1700           "intervalFactor": 2,
1701           "legendFormat": "{{instance}}",
1702           "metric": "jvm_memory_bytes_used",
1703           "refId": "A",
1704           "step": 4
1705         },
1706         {
1707           "datasource": {
1708             "type": "prometheus",
1709             "uid": "PBFA97CFB590B2093"
1710           },
1711           "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1712           "interval": "",
1713           "legendFormat": "{{instance}}",
1714           "refId": "B"
1715         }
1716       ],
1717       "title": "JVM Memory Used",
1718       "type": "timeseries"
1719     },
1720     {
1721       "datasource": {
1722         "type": "prometheus",
1723         "uid": "PBFA97CFB590B2093"
1724       },
1725       "fieldConfig": {
1726         "defaults": {
1727           "color": {
1728             "mode": "palette-classic"
1729           },
1730           "custom": {
1731             "axisCenteredZero": false,
1732             "axisColorMode": "text",
1733             "axisLabel": "% time in GC",
1734             "axisPlacement": "auto",
1735             "barAlignment": 0,
1736             "drawStyle": "line",
1737             "fillOpacity": 10,
1738             "gradientMode": "none",
1739             "hideFrom": {
1740               "legend": false,
1741               "tooltip": false,
1742               "viz": false
1743             },
1744             "lineInterpolation": "linear",
1745             "lineWidth": 2,
1746             "pointSize": 5,
1747             "scaleDistribution": {
1748               "type": "linear"
1749             },
1750             "showPoints": "never",
1751             "spanNulls": false,
1752             "stacking": {
1753               "group": "A",
1754               "mode": "none"
1755             },
1756             "thresholdsStyle": {
1757               "mode": "off"
1758             }
1759           },
1760           "links": [],
1761           "mappings": [],
1762           "thresholds": {
1763             "mode": "absolute",
1764             "steps": [
1765               {
1766                 "color": "green",
1767                 "value": null
1768               },
1769               {
1770                 "color": "red",
1771                 "value": 80
1772               }
1773             ]
1774           },
1775           "unit": "percentunit"
1776         },
1777         "overrides": [
1778           {
1779             "matcher": {
1780               "id": "byName",
1781               "options": "localhost:7071"
1782             },
1783             "properties": [
1784               {
1785                 "id": "color",
1786                 "value": {
1787                   "fixedColor": "#890F02",
1788                   "mode": "fixed"
1789                 }
1790               }
1791             ]
1792           }
1793         ]
1794       },
1795       "gridPos": {
1796         "h": 8,
1797         "w": 8,
1798         "x": 16,
1799         "y": 27
1800       },
1801       "id": 3,
1802       "links": [],
1803       "options": {
1804         "legend": {
1805           "calcs": [
1806             "mean",
1807             "lastNotNull",
1808             "max"
1809           ],
1810           "displayMode": "table",
1811           "placement": "bottom",
1812           "showLegend": true
1813         },
1814         "tooltip": {
1815           "mode": "single",
1816           "sort": "none"
1817         }
1818       },
1819       "pluginVersion": "8.1.3",
1820       "targets": [
1821         {
1822           "datasource": {
1823             "type": "prometheus",
1824             "uid": "PBFA97CFB590B2093"
1825           },
1826           "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1827           "interval": "",
1828           "intervalFactor": 2,
1829           "legendFormat": "{{instance}}",
1830           "metric": "jvm_gc_collection_seconds_sum",
1831           "refId": "A",
1832           "step": 4
1833         }
1834       ],
1835       "title": "Time spent in GC",
1836       "type": "timeseries"
1837     },
1838     {
1839       "collapsed": true,
1840       "datasource": {
1841         "type": "datasource",
1842         "uid": "grafana"
1843       },
1844       "gridPos": {
1845         "h": 1,
1846         "w": 24,
1847         "x": 0,
1848         "y": 35
1849       },
1850       "id": 29,
1851       "panels": [
1852         {
1853           "datasource": {
1854             "type": "prometheus",
1855             "uid": "PBFA97CFB590B2093"
1856           },
1857           "fieldConfig": {
1858             "defaults": {
1859               "color": {
1860                 "mode": "palette-classic"
1861               },
1862               "custom": {
1863                 "axisLabel": "Messages/s",
1864                 "axisPlacement": "auto",
1865                 "barAlignment": 0,
1866                 "drawStyle": "line",
1867                 "fillOpacity": 10,
1868                 "gradientMode": "none",
1869                 "hideFrom": {
1870                   "legend": false,
1871                   "tooltip": false,
1872                   "viz": false
1873                 },
1874                 "lineInterpolation": "linear",
1875                 "lineWidth": 2,
1876                 "pointSize": 5,
1877                 "scaleDistribution": {
1878                   "type": "linear"
1879                 },
1880                 "showPoints": "never",
1881                 "spanNulls": false,
1882                 "stacking": {
1883                   "group": "A",
1884                   "mode": "none"
1885                 },
1886                 "thresholdsStyle": {
1887                   "mode": "off"
1888                 }
1889               },
1890               "links": [],
1891               "mappings": [],
1892               "min": 0,
1893               "thresholds": {
1894                 "mode": "absolute",
1895                 "steps": [
1896                   {
1897                     "color": "green"
1898                   }
1899                 ]
1900               },
1901               "unit": "cps"
1902             },
1903             "overrides": []
1904           },
1905           "gridPos": {
1906             "h": 8,
1907             "w": 8,
1908             "x": 0,
1909             "y": 24
1910           },
1911           "id": 4,
1912           "links": [],
1913           "options": {
1914             "legend": {
1915               "calcs": [
1916                 "mean",
1917                 "lastNotNull",
1918                 "max"
1919               ],
1920               "displayMode": "table",
1921               "placement": "bottom",
1922               "showLegend": true
1923             },
1924             "tooltip": {
1925               "mode": "single"
1926             }
1927           },
1928           "pluginVersion": "8.1.3",
1929           "targets": [
1930             {
1931               "datasource": {
1932                 "type": "prometheus",
1933                 "uid": "PBFA97CFB590B2093"
1934               },
1935               "exemplar": true,
1936               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1937               "interval": "",
1938               "intervalFactor": 2,
1939               "legendFormat": "bytes/sec",
1940               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
1941               "refId": "A",
1942               "step": 4
1943             }
1944           ],
1945           "title": "Messages In",
1946           "type": "timeseries"
1947         },
1948         {
1949           "datasource": {
1950             "type": "prometheus",
1951             "uid": "PBFA97CFB590B2093"
1952           },
1953           "fieldConfig": {
1954             "defaults": {
1955               "color": {
1956                 "mode": "palette-classic"
1957               },
1958               "custom": {
1959                 "axisLabel": "Bytes/s",
1960                 "axisPlacement": "auto",
1961                 "barAlignment": 0,
1962                 "drawStyle": "line",
1963                 "fillOpacity": 10,
1964                 "gradientMode": "none",
1965                 "hideFrom": {
1966                   "legend": false,
1967                   "tooltip": false,
1968                   "viz": false
1969                 },
1970                 "lineInterpolation": "linear",
1971                 "lineWidth": 2,
1972                 "pointSize": 5,
1973                 "scaleDistribution": {
1974                   "type": "linear"
1975                 },
1976                 "showPoints": "never",
1977                 "spanNulls": false,
1978                 "stacking": {
1979                   "group": "A",
1980                   "mode": "none"
1981                 },
1982                 "thresholdsStyle": {
1983                   "mode": "off"
1984                 }
1985               },
1986               "links": [],
1987               "mappings": [],
1988               "min": 0,
1989               "thresholds": {
1990                 "mode": "absolute",
1991                 "steps": [
1992                   {
1993                     "color": "green"
1994                   },
1995                   {
1996                     "color": "red",
1997                     "value": 80
1998                   }
1999                 ]
2000               },
2001               "unit": "Bps"
2002             },
2003             "overrides": []
2004           },
2005           "gridPos": {
2006             "h": 8,
2007             "w": 8,
2008             "x": 8,
2009             "y": 24
2010           },
2011           "id": 5,
2012           "links": [],
2013           "options": {
2014             "legend": {
2015               "calcs": [
2016                 "mean",
2017                 "lastNotNull",
2018                 "max"
2019               ],
2020               "displayMode": "table",
2021               "placement": "bottom",
2022               "showLegend": true
2023             },
2024             "tooltip": {
2025               "mode": "single"
2026             }
2027           },
2028           "pluginVersion": "8.1.3",
2029           "targets": [
2030             {
2031               "datasource": {
2032                 "type": "prometheus",
2033                 "uid": "PBFA97CFB590B2093"
2034               },
2035               "exemplar": true,
2036               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2037               "format": "time_series",
2038               "interval": "",
2039               "intervalFactor": 2,
2040               "legendFormat": "bytes/sec",
2041               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2042               "refId": "A",
2043               "step": 4
2044             }
2045           ],
2046           "title": "Bytes In",
2047           "type": "timeseries"
2048         },
2049         {
2050           "datasource": {
2051             "type": "prometheus",
2052             "uid": "PBFA97CFB590B2093"
2053           },
2054           "fieldConfig": {
2055             "defaults": {
2056               "color": {
2057                 "mode": "palette-classic"
2058               },
2059               "custom": {
2060                 "axisLabel": "Bytes/s",
2061                 "axisPlacement": "auto",
2062                 "barAlignment": 0,
2063                 "drawStyle": "line",
2064                 "fillOpacity": 10,
2065                 "gradientMode": "none",
2066                 "hideFrom": {
2067                   "legend": false,
2068                   "tooltip": false,
2069                   "viz": false
2070                 },
2071                 "lineInterpolation": "linear",
2072                 "lineWidth": 2,
2073                 "pointSize": 5,
2074                 "scaleDistribution": {
2075                   "type": "linear"
2076                 },
2077                 "showPoints": "never",
2078                 "spanNulls": false,
2079                 "stacking": {
2080                   "group": "A",
2081                   "mode": "none"
2082                 },
2083                 "thresholdsStyle": {
2084                   "mode": "off"
2085                 }
2086               },
2087               "links": [],
2088               "mappings": [],
2089               "min": 0,
2090               "thresholds": {
2091                 "mode": "absolute",
2092                 "steps": [
2093                   {
2094                     "color": "green"
2095                   },
2096                   {
2097                     "color": "red",
2098                     "value": 80
2099                   }
2100                 ]
2101               },
2102               "unit": "Bps"
2103             },
2104             "overrides": []
2105           },
2106           "gridPos": {
2107             "h": 8,
2108             "w": 8,
2109             "x": 16,
2110             "y": 24
2111           },
2112           "id": 6,
2113           "links": [],
2114           "options": {
2115             "legend": {
2116               "calcs": [
2117                 "mean",
2118                 "lastNotNull",
2119                 "max"
2120               ],
2121               "displayMode": "table",
2122               "placement": "bottom",
2123               "showLegend": true
2124             },
2125             "tooltip": {
2126               "mode": "single"
2127             }
2128           },
2129           "pluginVersion": "8.1.3",
2130           "targets": [
2131             {
2132               "datasource": {
2133                 "type": "prometheus",
2134                 "uid": "PBFA97CFB590B2093"
2135               },
2136               "exemplar": true,
2137               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2138               "interval": "",
2139               "intervalFactor": 2,
2140               "legendFormat": "bytes/sec",
2141               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2142               "refId": "A",
2143               "step": 4
2144             }
2145           ],
2146           "title": "Bytes Out",
2147           "type": "timeseries"
2148         },
2149         {
2150           "datasource": {
2151             "type": "prometheus",
2152             "uid": "PBFA97CFB590B2093"
2153           },
2154           "fieldConfig": {
2155             "defaults": {
2156               "color": {
2157                 "mode": "palette-classic"
2158               },
2159               "custom": {
2160                 "axisLabel": "Messages/s",
2161                 "axisPlacement": "auto",
2162                 "barAlignment": 0,
2163                 "drawStyle": "line",
2164                 "fillOpacity": 10,
2165                 "gradientMode": "none",
2166                 "hideFrom": {
2167                   "legend": false,
2168                   "tooltip": false,
2169                   "viz": false
2170                 },
2171                 "lineInterpolation": "linear",
2172                 "lineWidth": 2,
2173                 "pointSize": 5,
2174                 "scaleDistribution": {
2175                   "type": "linear"
2176                 },
2177                 "showPoints": "never",
2178                 "spanNulls": false,
2179                 "stacking": {
2180                   "group": "A",
2181                   "mode": "none"
2182                 },
2183                 "thresholdsStyle": {
2184                   "mode": "off"
2185                 }
2186               },
2187               "links": [],
2188               "mappings": [],
2189               "thresholds": {
2190                 "mode": "absolute",
2191                 "steps": [
2192                   {
2193                     "color": "green"
2194                   },
2195                   {
2196                     "color": "red",
2197                     "value": 80
2198                   }
2199                 ]
2200               },
2201               "unit": "cps"
2202             },
2203             "overrides": []
2204           },
2205           "gridPos": {
2206             "h": 8,
2207             "w": 8,
2208             "x": 0,
2209             "y": 32
2210           },
2211           "id": 10,
2212           "links": [],
2213           "options": {
2214             "legend": {
2215               "calcs": [
2216                 "mean",
2217                 "lastNotNull",
2218                 "max"
2219               ],
2220               "displayMode": "table",
2221               "placement": "bottom",
2222               "showLegend": true
2223             },
2224             "tooltip": {
2225               "mode": "single"
2226             }
2227           },
2228           "pluginVersion": "8.1.3",
2229           "targets": [
2230             {
2231               "datasource": {
2232                 "type": "prometheus",
2233                 "uid": "PBFA97CFB590B2093"
2234               },
2235               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2236               "format": "time_series",
2237               "interval": "",
2238               "intervalFactor": 2,
2239               "legendFormat": "{{instance}}",
2240               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2241               "refId": "A",
2242               "step": 4
2243             }
2244           ],
2245           "title": "Messages In Per Broker",
2246           "type": "timeseries"
2247         },
2248         {
2249           "datasource": {
2250             "type": "prometheus",
2251             "uid": "PBFA97CFB590B2093"
2252           },
2253           "fieldConfig": {
2254             "defaults": {
2255               "color": {
2256                 "mode": "palette-classic"
2257               },
2258               "custom": {
2259                 "axisLabel": "Bytes/s",
2260                 "axisPlacement": "auto",
2261                 "barAlignment": 0,
2262                 "drawStyle": "line",
2263                 "fillOpacity": 10,
2264                 "gradientMode": "none",
2265                 "hideFrom": {
2266                   "legend": false,
2267                   "tooltip": false,
2268                   "viz": false
2269                 },
2270                 "lineInterpolation": "linear",
2271                 "lineWidth": 2,
2272                 "pointSize": 5,
2273                 "scaleDistribution": {
2274                   "type": "linear"
2275                 },
2276                 "showPoints": "never",
2277                 "spanNulls": false,
2278                 "stacking": {
2279                   "group": "A",
2280                   "mode": "none"
2281                 },
2282                 "thresholdsStyle": {
2283                   "mode": "off"
2284                 }
2285               },
2286               "links": [],
2287               "mappings": [],
2288               "thresholds": {
2289                 "mode": "absolute",
2290                 "steps": [
2291                   {
2292                     "color": "green"
2293                   },
2294                   {
2295                     "color": "red",
2296                     "value": 80
2297                   }
2298                 ]
2299               },
2300               "unit": "Bps"
2301             },
2302             "overrides": []
2303           },
2304           "gridPos": {
2305             "h": 8,
2306             "w": 8,
2307             "x": 8,
2308             "y": 32
2309           },
2310           "id": 7,
2311           "links": [],
2312           "options": {
2313             "legend": {
2314               "calcs": [
2315                 "mean",
2316                 "lastNotNull",
2317                 "max"
2318               ],
2319               "displayMode": "table",
2320               "placement": "bottom",
2321               "showLegend": true
2322             },
2323             "tooltip": {
2324               "mode": "single"
2325             }
2326           },
2327           "pluginVersion": "8.1.3",
2328           "targets": [
2329             {
2330               "datasource": {
2331                 "type": "prometheus",
2332                 "uid": "PBFA97CFB590B2093"
2333               },
2334               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2335               "format": "time_series",
2336               "interval": "",
2337               "intervalFactor": 2,
2338               "legendFormat": "{{instance}}",
2339               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2340               "refId": "A",
2341               "step": 4
2342             }
2343           ],
2344           "title": "Bytes In Per Broker",
2345           "type": "timeseries"
2346         },
2347         {
2348           "datasource": {
2349             "type": "prometheus",
2350             "uid": "PBFA97CFB590B2093"
2351           },
2352           "fieldConfig": {
2353             "defaults": {
2354               "color": {
2355                 "mode": "palette-classic"
2356               },
2357               "custom": {
2358                 "axisLabel": "",
2359                 "axisPlacement": "auto",
2360                 "barAlignment": 0,
2361                 "drawStyle": "line",
2362                 "fillOpacity": 10,
2363                 "gradientMode": "none",
2364                 "hideFrom": {
2365                   "legend": false,
2366                   "tooltip": false,
2367                   "viz": false
2368                 },
2369                 "lineInterpolation": "linear",
2370                 "lineWidth": 1,
2371                 "pointSize": 5,
2372                 "scaleDistribution": {
2373                   "type": "linear"
2374                 },
2375                 "showPoints": "never",
2376                 "spanNulls": true,
2377                 "stacking": {
2378                   "group": "A",
2379                   "mode": "none"
2380                 },
2381                 "thresholdsStyle": {
2382                   "mode": "off"
2383                 }
2384               },
2385               "links": [],
2386               "mappings": [],
2387               "min": 0,
2388               "thresholds": {
2389                 "mode": "absolute",
2390                 "steps": [
2391                   {
2392                     "color": "green"
2393                   },
2394                   {
2395                     "color": "red",
2396                     "value": 80
2397                   }
2398                 ]
2399               },
2400               "unit": "binBps"
2401             },
2402             "overrides": []
2403           },
2404           "gridPos": {
2405             "h": 8,
2406             "w": 8,
2407             "x": 16,
2408             "y": 32
2409           },
2410           "id": 9,
2411           "links": [],
2412           "options": {
2413             "legend": {
2414               "calcs": [
2415                 "mean",
2416                 "lastNotNull",
2417                 "max"
2418               ],
2419               "displayMode": "table",
2420               "placement": "bottom",
2421               "showLegend": true
2422             },
2423             "tooltip": {
2424               "mode": "single"
2425             }
2426           },
2427           "pluginVersion": "8.1.3",
2428           "targets": [
2429             {
2430               "datasource": {
2431                 "type": "prometheus",
2432                 "uid": "PBFA97CFB590B2093"
2433               },
2434               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2435               "format": "time_series",
2436               "interval": "",
2437               "intervalFactor": 1,
2438               "legendFormat": "{{instance}}",
2439               "refId": "A"
2440             }
2441           ],
2442           "title": "Bytes Out Per Broker",
2443           "type": "timeseries"
2444         }
2445       ],
2446       "targets": [
2447         {
2448           "datasource": {
2449             "type": "datasource",
2450             "uid": "grafana"
2451           },
2452           "refId": "A"
2453         }
2454       ],
2455       "title": "Throughput In/Out",
2456       "type": "row"
2457     },
2458     {
2459       "collapsed": true,
2460       "datasource": {
2461         "type": "datasource",
2462         "uid": "grafana"
2463       },
2464       "gridPos": {
2465         "h": 1,
2466         "w": 24,
2467         "x": 0,
2468         "y": 36
2469       },
2470       "id": 44,
2471       "panels": [
2472         {
2473           "datasource": {
2474             "type": "prometheus",
2475             "uid": "PBFA97CFB590B2093"
2476           },
2477           "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2478           "fieldConfig": {
2479             "defaults": {
2480               "color": {
2481                 "mode": "palette-classic"
2482               },
2483               "custom": {
2484                 "axisLabel": "",
2485                 "axisPlacement": "auto",
2486                 "barAlignment": 0,
2487                 "drawStyle": "line",
2488                 "fillOpacity": 10,
2489                 "gradientMode": "none",
2490                 "hideFrom": {
2491                   "legend": false,
2492                   "tooltip": false,
2493                   "viz": false
2494                 },
2495                 "lineInterpolation": "linear",
2496                 "lineWidth": 1,
2497                 "pointSize": 5,
2498                 "scaleDistribution": {
2499                   "type": "linear"
2500                 },
2501                 "showPoints": "never",
2502                 "spanNulls": true,
2503                 "stacking": {
2504                   "group": "A",
2505                   "mode": "none"
2506                 },
2507                 "thresholdsStyle": {
2508                   "mode": "off"
2509                 }
2510               },
2511               "links": [],
2512               "mappings": [],
2513               "thresholds": {
2514                 "mode": "absolute",
2515                 "steps": [
2516                   {
2517                     "color": "green"
2518                   },
2519                   {
2520                     "color": "red",
2521                     "value": 80
2522                   }
2523                 ]
2524               },
2525               "unit": "percentunit"
2526             },
2527             "overrides": []
2528           },
2529           "gridPos": {
2530             "h": 9,
2531             "w": 12,
2532             "x": 0,
2533             "y": 25
2534           },
2535           "id": 24,
2536           "options": {
2537             "legend": {
2538               "calcs": [
2539                 "mean",
2540                 "lastNotNull",
2541                 "max"
2542               ],
2543               "displayMode": "table",
2544               "placement": "bottom",
2545               "showLegend": true
2546             },
2547             "tooltip": {
2548               "mode": "single"
2549             }
2550           },
2551           "pluginVersion": "8.1.3",
2552           "targets": [
2553             {
2554               "datasource": {
2555                 "type": "prometheus",
2556                 "uid": "PBFA97CFB590B2093"
2557               },
2558               "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2559               "interval": "",
2560               "legendFormat": "{{instance}}",
2561               "refId": "A"
2562             }
2563           ],
2564           "title": "Network Processor Avg Usage Percent",
2565           "type": "timeseries"
2566         },
2567         {
2568           "datasource": {
2569             "type": "prometheus",
2570             "uid": "PBFA97CFB590B2093"
2571           },
2572           "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2573           "fieldConfig": {
2574             "defaults": {
2575               "color": {
2576                 "mode": "palette-classic"
2577               },
2578               "custom": {
2579                 "axisLabel": "",
2580                 "axisPlacement": "auto",
2581                 "barAlignment": 0,
2582                 "drawStyle": "line",
2583                 "fillOpacity": 10,
2584                 "gradientMode": "none",
2585                 "hideFrom": {
2586                   "legend": false,
2587                   "tooltip": false,
2588                   "viz": false
2589                 },
2590                 "lineInterpolation": "linear",
2591                 "lineWidth": 1,
2592                 "pointSize": 5,
2593                 "scaleDistribution": {
2594                   "type": "linear"
2595                 },
2596                 "showPoints": "never",
2597                 "spanNulls": true,
2598                 "stacking": {
2599                   "group": "A",
2600                   "mode": "none"
2601                 },
2602                 "thresholdsStyle": {
2603                   "mode": "off"
2604                 }
2605               },
2606               "links": [],
2607               "mappings": [],
2608               "min": 0,
2609               "thresholds": {
2610                 "mode": "absolute",
2611                 "steps": [
2612                   {
2613                     "color": "green"
2614                   },
2615                   {
2616                     "color": "red",
2617                     "value": 80
2618                   }
2619                 ]
2620               },
2621               "unit": "percentunit"
2622             },
2623             "overrides": []
2624           },
2625           "gridPos": {
2626             "h": 9,
2627             "w": 12,
2628             "x": 12,
2629             "y": 25
2630           },
2631           "id": 25,
2632           "options": {
2633             "legend": {
2634               "calcs": [
2635                 "mean",
2636                 "lastNotNull",
2637                 "max"
2638               ],
2639               "displayMode": "table",
2640               "placement": "bottom",
2641               "showLegend": true
2642             },
2643             "tooltip": {
2644               "mode": "single"
2645             }
2646           },
2647           "pluginVersion": "8.1.3",
2648           "targets": [
2649             {
2650               "datasource": {
2651                 "type": "prometheus",
2652                 "uid": "PBFA97CFB590B2093"
2653               },
2654               "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2655               "interval": "",
2656               "legendFormat": "{{instance}}",
2657               "refId": "A"
2658             }
2659           ],
2660           "title": "Request Handler Avg Percent",
2661           "type": "timeseries"
2662         }
2663       ],
2664       "targets": [
2665         {
2666           "datasource": {
2667             "type": "datasource",
2668             "uid": "grafana"
2669           },
2670           "refId": "A"
2671         }
2672       ],
2673       "title": "Thread utilization",
2674       "type": "row"
2675     },
2676     {
2677       "collapsed": false,
2678       "datasource": {
2679         "type": "datasource",
2680         "uid": "grafana"
2681       },
2682       "gridPos": {
2683         "h": 1,
2684         "w": 24,
2685         "x": 0,
2686         "y": 38
2687       },
2688       "id": 82,
2689       "panels": [],
2690       "targets": [
2691         {
2692           "datasource": {
2693             "type": "datasource",
2694             "uid": "grafana"
2695           },
2696           "refId": "A"
2697         }
2698       ],
2699       "title": "Isr Shrinks / Expands",
2700       "type": "row"
2701     },
2702     {
2703       "datasource": {
2704         "type": "prometheus",
2705         "uid": "PBFA97CFB590B2093"
2706       },
2707       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2708       "fieldConfig": {
2709         "defaults": {
2710           "color": {
2711             "mode": "palette-classic"
2712           },
2713           "custom": {
2714             "axisCenteredZero": false,
2715             "axisColorMode": "text",
2716             "axisLabel": "",
2717             "axisPlacement": "auto",
2718             "barAlignment": 0,
2719             "drawStyle": "line",
2720             "fillOpacity": 10,
2721             "gradientMode": "none",
2722             "hideFrom": {
2723               "legend": false,
2724               "tooltip": false,
2725               "viz": false
2726             },
2727             "lineInterpolation": "linear",
2728             "lineWidth": 1,
2729             "pointSize": 5,
2730             "scaleDistribution": {
2731               "type": "linear"
2732             },
2733             "showPoints": "never",
2734             "spanNulls": true,
2735             "stacking": {
2736               "group": "A",
2737               "mode": "none"
2738             },
2739             "thresholdsStyle": {
2740               "mode": "off"
2741             }
2742           },
2743           "links": [],
2744           "mappings": [],
2745           "min": 0,
2746           "thresholds": {
2747             "mode": "absolute",
2748             "steps": [
2749               {
2750                 "color": "green",
2751                 "value": null
2752               },
2753               {
2754                 "color": "red",
2755                 "value": 80
2756               }
2757             ]
2758           },
2759           "unit": "ops"
2760         },
2761         "overrides": []
2762       },
2763       "gridPos": {
2764         "h": 9,
2765         "w": 12,
2766         "x": 0,
2767         "y": 39
2768       },
2769       "id": 80,
2770       "options": {
2771         "legend": {
2772           "calcs": [
2773             "mean",
2774             "lastNotNull",
2775             "max"
2776           ],
2777           "displayMode": "table",
2778           "placement": "bottom",
2779           "showLegend": true
2780         },
2781         "tooltip": {
2782           "mode": "single",
2783           "sort": "none"
2784         }
2785       },
2786       "pluginVersion": "8.1.3",
2787       "targets": [
2788         {
2789           "datasource": {
2790             "type": "prometheus",
2791             "uid": "PBFA97CFB590B2093"
2792           },
2793           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2794           "interval": "",
2795           "legendFormat": "{{instance}}",
2796           "refId": "A"
2797         }
2798       ],
2799       "title": "IsrShrinks per Sec",
2800       "type": "timeseries"
2801     },
2802     {
2803       "datasource": {
2804         "type": "prometheus",
2805         "uid": "PBFA97CFB590B2093"
2806       },
2807       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2808       "fieldConfig": {
2809         "defaults": {
2810           "color": {
2811             "mode": "palette-classic"
2812           },
2813           "custom": {
2814             "axisCenteredZero": false,
2815             "axisColorMode": "text",
2816             "axisLabel": "",
2817             "axisPlacement": "auto",
2818             "barAlignment": 0,
2819             "drawStyle": "line",
2820             "fillOpacity": 10,
2821             "gradientMode": "none",
2822             "hideFrom": {
2823               "legend": false,
2824               "tooltip": false,
2825               "viz": false
2826             },
2827             "lineInterpolation": "linear",
2828             "lineWidth": 1,
2829             "pointSize": 5,
2830             "scaleDistribution": {
2831               "type": "linear"
2832             },
2833             "showPoints": "never",
2834             "spanNulls": true,
2835             "stacking": {
2836               "group": "A",
2837               "mode": "none"
2838             },
2839             "thresholdsStyle": {
2840               "mode": "off"
2841             }
2842           },
2843           "links": [],
2844           "mappings": [],
2845           "min": 0,
2846           "thresholds": {
2847             "mode": "absolute",
2848             "steps": [
2849               {
2850                 "color": "green",
2851                 "value": null
2852               },
2853               {
2854                 "color": "red",
2855                 "value": 80
2856               }
2857             ]
2858           },
2859           "unit": "ops"
2860         },
2861         "overrides": []
2862       },
2863       "gridPos": {
2864         "h": 9,
2865         "w": 12,
2866         "x": 12,
2867         "y": 39
2868       },
2869       "id": 83,
2870       "options": {
2871         "legend": {
2872           "calcs": [
2873             "mean",
2874             "lastNotNull",
2875             "max"
2876           ],
2877           "displayMode": "table",
2878           "placement": "bottom",
2879           "showLegend": true
2880         },
2881         "tooltip": {
2882           "mode": "single",
2883           "sort": "none"
2884         }
2885       },
2886       "pluginVersion": "8.1.3",
2887       "targets": [
2888         {
2889           "datasource": {
2890             "type": "prometheus",
2891             "uid": "PBFA97CFB590B2093"
2892           },
2893           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2894           "hide": false,
2895           "interval": "",
2896           "legendFormat": "{{instance}}",
2897           "refId": "A"
2898         }
2899       ],
2900       "title": "IsrExpands per Sec",
2901       "type": "timeseries"
2902     },
2903     {
2904       "collapsed": true,
2905       "datasource": {
2906         "type": "datasource",
2907         "uid": "grafana"
2908       },
2909       "gridPos": {
2910         "h": 1,
2911         "w": 24,
2912         "x": 0,
2913         "y": 48
2914       },
2915       "id": 53,
2916       "panels": [
2917         {
2918           "datasource": {
2919             "type": "prometheus",
2920             "uid": "PBFA97CFB590B2093"
2921           },
2922           "fieldConfig": {
2923             "defaults": {
2924               "color": {
2925                 "mode": "palette-classic"
2926               },
2927               "custom": {
2928                 "axisLabel": "",
2929                 "axisPlacement": "auto",
2930                 "barAlignment": 0,
2931                 "drawStyle": "line",
2932                 "fillOpacity": 10,
2933                 "gradientMode": "none",
2934                 "hideFrom": {
2935                   "legend": false,
2936                   "tooltip": false,
2937                   "viz": false
2938                 },
2939                 "lineInterpolation": "linear",
2940                 "lineWidth": 1,
2941                 "pointSize": 5,
2942                 "scaleDistribution": {
2943                   "type": "linear"
2944                 },
2945                 "showPoints": "never",
2946                 "spanNulls": true,
2947                 "stacking": {
2948                   "group": "A",
2949                   "mode": "none"
2950                 },
2951                 "thresholdsStyle": {
2952                   "mode": "off"
2953                 }
2954               },
2955               "links": [],
2956               "mappings": [],
2957               "thresholds": {
2958                 "mode": "absolute",
2959                 "steps": [
2960                   {
2961                     "color": "green"
2962                   },
2963                   {
2964                     "color": "red",
2965                     "value": 80
2966                   }
2967                 ]
2968               },
2969               "unit": "bytes"
2970             },
2971             "overrides": []
2972           },
2973           "gridPos": {
2974             "h": 9,
2975             "w": 12,
2976             "x": 0,
2977             "y": 28
2978           },
2979           "id": 55,
2980           "options": {
2981             "legend": {
2982               "calcs": [
2983                 "mean",
2984                 "lastNotNull",
2985                 "max"
2986               ],
2987               "displayMode": "table",
2988               "placement": "bottom",
2989               "showLegend": true
2990             },
2991             "tooltip": {
2992               "mode": "single"
2993             }
2994           },
2995           "pluginVersion": "8.1.3",
2996           "targets": [
2997             {
2998               "datasource": {
2999                 "type": "prometheus",
3000                 "uid": "PBFA97CFB590B2093"
3001               },
3002               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3003               "legendFormat": "{{topic}}",
3004               "refId": "A"
3005             }
3006           ],
3007           "title": "Log size per Topic",
3008           "type": "timeseries"
3009         },
3010         {
3011           "datasource": {
3012             "type": "prometheus",
3013             "uid": "PBFA97CFB590B2093"
3014           },
3015           "fieldConfig": {
3016             "defaults": {
3017               "color": {
3018                 "mode": "palette-classic"
3019               },
3020               "custom": {
3021                 "axisLabel": "",
3022                 "axisPlacement": "auto",
3023                 "barAlignment": 0,
3024                 "drawStyle": "line",
3025                 "fillOpacity": 10,
3026                 "gradientMode": "none",
3027                 "hideFrom": {
3028                   "legend": false,
3029                   "tooltip": false,
3030                   "viz": false
3031                 },
3032                 "lineInterpolation": "linear",
3033                 "lineWidth": 1,
3034                 "pointSize": 5,
3035                 "scaleDistribution": {
3036                   "type": "linear"
3037                 },
3038                 "showPoints": "never",
3039                 "spanNulls": true,
3040                 "stacking": {
3041                   "group": "A",
3042                   "mode": "none"
3043                 },
3044                 "thresholdsStyle": {
3045                   "mode": "off"
3046                 }
3047               },
3048               "links": [],
3049               "mappings": [],
3050               "thresholds": {
3051                 "mode": "absolute",
3052                 "steps": [
3053                   {
3054                     "color": "green"
3055                   },
3056                   {
3057                     "color": "red",
3058                     "value": 80
3059                   }
3060                 ]
3061               },
3062               "unit": "bytes"
3063             },
3064             "overrides": []
3065           },
3066           "gridPos": {
3067             "h": 9,
3068             "w": 12,
3069             "x": 12,
3070             "y": 28
3071           },
3072           "id": 56,
3073           "options": {
3074             "legend": {
3075               "calcs": [
3076                 "mean",
3077                 "lastNotNull",
3078                 "max"
3079               ],
3080               "displayMode": "table",
3081               "placement": "bottom",
3082               "showLegend": true
3083             },
3084             "tooltip": {
3085               "mode": "single"
3086             }
3087           },
3088           "pluginVersion": "8.1.3",
3089           "targets": [
3090             {
3091               "datasource": {
3092                 "type": "prometheus",
3093                 "uid": "PBFA97CFB590B2093"
3094               },
3095               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3096               "legendFormat": "{{instance}}",
3097               "refId": "A"
3098             }
3099           ],
3100           "title": "Log size per Broker",
3101           "type": "timeseries"
3102         }
3103       ],
3104       "targets": [
3105         {
3106           "datasource": {
3107             "type": "datasource",
3108             "uid": "grafana"
3109           },
3110           "refId": "A"
3111         }
3112       ],
3113       "title": "Logs size",
3114       "type": "row"
3115     },
3116     {
3117       "collapsed": true,
3118       "datasource": {
3119         "type": "datasource",
3120         "uid": "grafana"
3121       },
3122       "gridPos": {
3123         "h": 1,
3124         "w": 24,
3125         "x": 0,
3126         "y": 49
3127       },
3128       "id": 58,
3129       "panels": [
3130         {
3131           "datasource": {
3132             "type": "prometheus",
3133             "uid": "PBFA97CFB590B2093"
3134           },
3135           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3136           "fieldConfig": {
3137             "defaults": {
3138               "color": {
3139                 "mode": "palette-classic"
3140               },
3141               "custom": {
3142                 "axisLabel": "",
3143                 "axisPlacement": "auto",
3144                 "barAlignment": 0,
3145                 "drawStyle": "line",
3146                 "fillOpacity": 10,
3147                 "gradientMode": "none",
3148                 "hideFrom": {
3149                   "legend": false,
3150                   "tooltip": false,
3151                   "viz": false
3152                 },
3153                 "lineInterpolation": "linear",
3154                 "lineWidth": 1,
3155                 "pointSize": 5,
3156                 "scaleDistribution": {
3157                   "type": "linear"
3158                 },
3159                 "showPoints": "never",
3160                 "spanNulls": true,
3161                 "stacking": {
3162                   "group": "A",
3163                   "mode": "none"
3164                 },
3165                 "thresholdsStyle": {
3166                   "mode": "off"
3167                 }
3168               },
3169               "links": [],
3170               "mappings": [],
3171               "thresholds": {
3172                 "mode": "absolute",
3173                 "steps": [
3174                   {
3175                     "color": "green"
3176                   },
3177                   {
3178                     "color": "red",
3179                     "value": 80
3180                   }
3181                 ]
3182               },
3183               "unit": "ms"
3184             },
3185             "overrides": []
3186           },
3187           "gridPos": {
3188             "h": 9,
3189             "w": 12,
3190             "x": 0,
3191             "y": 29
3192           },
3193           "id": 60,
3194           "options": {
3195             "legend": {
3196               "calcs": [
3197                 "mean",
3198                 "lastNotNull",
3199                 "max"
3200               ],
3201               "displayMode": "list",
3202               "placement": "bottom",
3203               "showLegend": true
3204             },
3205             "tooltip": {
3206               "mode": "single"
3207             }
3208           },
3209           "pluginVersion": "8.1.3",
3210           "targets": [
3211             {
3212               "datasource": {
3213                 "type": "prometheus",
3214                 "uid": "PBFA97CFB590B2093"
3215               },
3216               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3217               "hide": false,
3218               "legendFormat": "{{instance}} - {{quantile}}",
3219               "refId": "A"
3220             }
3221           ],
3222           "title": "Producer - RequestQueueTimeMs",
3223           "type": "timeseries"
3224         },
3225         {
3226           "datasource": {
3227             "type": "prometheus",
3228             "uid": "PBFA97CFB590B2093"
3229           },
3230           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3231           "fieldConfig": {
3232             "defaults": {
3233               "color": {
3234                 "mode": "palette-classic"
3235               },
3236               "custom": {
3237                 "axisLabel": "",
3238                 "axisPlacement": "auto",
3239                 "barAlignment": 0,
3240                 "drawStyle": "line",
3241                 "fillOpacity": 10,
3242                 "gradientMode": "none",
3243                 "hideFrom": {
3244                   "legend": false,
3245                   "tooltip": false,
3246                   "viz": false
3247                 },
3248                 "lineInterpolation": "linear",
3249                 "lineWidth": 1,
3250                 "pointSize": 5,
3251                 "scaleDistribution": {
3252                   "type": "linear"
3253                 },
3254                 "showPoints": "never",
3255                 "spanNulls": true,
3256                 "stacking": {
3257                   "group": "A",
3258                   "mode": "none"
3259                 },
3260                 "thresholdsStyle": {
3261                   "mode": "off"
3262                 }
3263               },
3264               "links": [],
3265               "mappings": [],
3266               "thresholds": {
3267                 "mode": "absolute",
3268                 "steps": [
3269                   {
3270                     "color": "green"
3271                   },
3272                   {
3273                     "color": "red",
3274                     "value": 80
3275                   }
3276                 ]
3277               },
3278               "unit": "ms"
3279             },
3280             "overrides": []
3281           },
3282           "gridPos": {
3283             "h": 9,
3284             "w": 12,
3285             "x": 12,
3286             "y": 29
3287           },
3288           "id": 61,
3289           "options": {
3290             "legend": {
3291               "calcs": [
3292                 "mean",
3293                 "lastNotNull",
3294                 "max"
3295               ],
3296               "displayMode": "list",
3297               "placement": "bottom",
3298               "showLegend": true
3299             },
3300             "tooltip": {
3301               "mode": "single"
3302             }
3303           },
3304           "pluginVersion": "8.1.3",
3305           "targets": [
3306             {
3307               "datasource": {
3308                 "type": "prometheus",
3309                 "uid": "PBFA97CFB590B2093"
3310               },
3311               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3312               "hide": false,
3313               "legendFormat": "{{instance}} - {{quantile}}",
3314               "refId": "A"
3315             }
3316           ],
3317           "title": "Producer - LocalTimeMs",
3318           "type": "timeseries"
3319         },
3320         {
3321           "datasource": {
3322             "type": "prometheus",
3323             "uid": "PBFA97CFB590B2093"
3324           },
3325           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3326           "fieldConfig": {
3327             "defaults": {
3328               "color": {
3329                 "mode": "palette-classic"
3330               },
3331               "custom": {
3332                 "axisLabel": "",
3333                 "axisPlacement": "auto",
3334                 "barAlignment": 0,
3335                 "drawStyle": "line",
3336                 "fillOpacity": 10,
3337                 "gradientMode": "none",
3338                 "hideFrom": {
3339                   "legend": false,
3340                   "tooltip": false,
3341                   "viz": false
3342                 },
3343                 "lineInterpolation": "linear",
3344                 "lineWidth": 1,
3345                 "pointSize": 5,
3346                 "scaleDistribution": {
3347                   "type": "linear"
3348                 },
3349                 "showPoints": "never",
3350                 "spanNulls": true,
3351                 "stacking": {
3352                   "group": "A",
3353                   "mode": "none"
3354                 },
3355                 "thresholdsStyle": {
3356                   "mode": "off"
3357                 }
3358               },
3359               "links": [],
3360               "mappings": [],
3361               "thresholds": {
3362                 "mode": "absolute",
3363                 "steps": [
3364                   {
3365                     "color": "green"
3366                   },
3367                   {
3368                     "color": "red",
3369                     "value": 80
3370                   }
3371                 ]
3372               },
3373               "unit": "ms"
3374             },
3375             "overrides": []
3376           },
3377           "gridPos": {
3378             "h": 10,
3379             "w": 8,
3380             "x": 0,
3381             "y": 38
3382           },
3383           "id": 62,
3384           "options": {
3385             "legend": {
3386               "calcs": [
3387                 "mean",
3388                 "lastNotNull",
3389                 "max"
3390               ],
3391               "displayMode": "table",
3392               "placement": "bottom",
3393               "showLegend": true
3394             },
3395             "tooltip": {
3396               "mode": "single"
3397             }
3398           },
3399           "pluginVersion": "8.1.3",
3400           "targets": [
3401             {
3402               "datasource": {
3403                 "type": "prometheus",
3404                 "uid": "PBFA97CFB590B2093"
3405               },
3406               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3407               "hide": false,
3408               "legendFormat": "{{instance}} - {{quantile}}",
3409               "refId": "A"
3410             }
3411           ],
3412           "title": "Producer - RemoteTimeMs",
3413           "type": "timeseries"
3414         },
3415         {
3416           "datasource": {
3417             "type": "prometheus",
3418             "uid": "PBFA97CFB590B2093"
3419           },
3420           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3421           "fieldConfig": {
3422             "defaults": {
3423               "color": {
3424                 "mode": "palette-classic"
3425               },
3426               "custom": {
3427                 "axisLabel": "",
3428                 "axisPlacement": "auto",
3429                 "barAlignment": 0,
3430                 "drawStyle": "line",
3431                 "fillOpacity": 10,
3432                 "gradientMode": "none",
3433                 "hideFrom": {
3434                   "legend": false,
3435                   "tooltip": false,
3436                   "viz": false
3437                 },
3438                 "lineInterpolation": "linear",
3439                 "lineWidth": 1,
3440                 "pointSize": 5,
3441                 "scaleDistribution": {
3442                   "type": "linear"
3443                 },
3444                 "showPoints": "never",
3445                 "spanNulls": true,
3446                 "stacking": {
3447                   "group": "A",
3448                   "mode": "none"
3449                 },
3450                 "thresholdsStyle": {
3451                   "mode": "off"
3452                 }
3453               },
3454               "links": [],
3455               "mappings": [],
3456               "thresholds": {
3457                 "mode": "absolute",
3458                 "steps": [
3459                   {
3460                     "color": "green"
3461                   },
3462                   {
3463                     "color": "red",
3464                     "value": 80
3465                   }
3466                 ]
3467               },
3468               "unit": "ms"
3469             },
3470             "overrides": []
3471           },
3472           "gridPos": {
3473             "h": 10,
3474             "w": 8,
3475             "x": 8,
3476             "y": 38
3477           },
3478           "id": 63,
3479           "options": {
3480             "legend": {
3481               "calcs": [
3482                 "mean",
3483                 "lastNotNull",
3484                 "max"
3485               ],
3486               "displayMode": "table",
3487               "placement": "bottom",
3488               "showLegend": true
3489             },
3490             "tooltip": {
3491               "mode": "single"
3492             }
3493           },
3494           "pluginVersion": "8.1.3",
3495           "targets": [
3496             {
3497               "datasource": {
3498                 "type": "prometheus",
3499                 "uid": "PBFA97CFB590B2093"
3500               },
3501               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3502               "hide": false,
3503               "legendFormat": "{{instance}} - {{quantile}}",
3504               "refId": "A"
3505             }
3506           ],
3507           "title": "Producer - ResponseQueueTimeMs",
3508           "type": "timeseries"
3509         },
3510         {
3511           "datasource": {
3512             "type": "prometheus",
3513             "uid": "PBFA97CFB590B2093"
3514           },
3515           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3516           "fieldConfig": {
3517             "defaults": {
3518               "color": {
3519                 "mode": "palette-classic"
3520               },
3521               "custom": {
3522                 "axisLabel": "",
3523                 "axisPlacement": "auto",
3524                 "barAlignment": 0,
3525                 "drawStyle": "line",
3526                 "fillOpacity": 10,
3527                 "gradientMode": "none",
3528                 "hideFrom": {
3529                   "legend": false,
3530                   "tooltip": false,
3531                   "viz": false
3532                 },
3533                 "lineInterpolation": "linear",
3534                 "lineWidth": 1,
3535                 "pointSize": 5,
3536                 "scaleDistribution": {
3537                   "type": "linear"
3538                 },
3539                 "showPoints": "never",
3540                 "spanNulls": true,
3541                 "stacking": {
3542                   "group": "A",
3543                   "mode": "none"
3544                 },
3545                 "thresholdsStyle": {
3546                   "mode": "off"
3547                 }
3548               },
3549               "links": [],
3550               "mappings": [],
3551               "min": 0,
3552               "thresholds": {
3553                 "mode": "absolute",
3554                 "steps": [
3555                   {
3556                     "color": "green"
3557                   },
3558                   {
3559                     "color": "red",
3560                     "value": 80
3561                   }
3562                 ]
3563               },
3564               "unit": "ms"
3565             },
3566             "overrides": []
3567           },
3568           "gridPos": {
3569             "h": 10,
3570             "w": 8,
3571             "x": 16,
3572             "y": 38
3573           },
3574           "id": 64,
3575           "options": {
3576             "legend": {
3577               "calcs": [
3578                 "mean",
3579                 "lastNotNull",
3580                 "max"
3581               ],
3582               "displayMode": "table",
3583               "placement": "bottom",
3584               "showLegend": true
3585             },
3586             "tooltip": {
3587               "mode": "single"
3588             }
3589           },
3590           "pluginVersion": "8.1.3",
3591           "targets": [
3592             {
3593               "datasource": {
3594                 "type": "prometheus",
3595                 "uid": "PBFA97CFB590B2093"
3596               },
3597               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3598               "hide": false,
3599               "legendFormat": "{{instance}} - {{quantile}}",
3600               "refId": "A"
3601             }
3602           ],
3603           "title": "Producer - ResponseSendTimeMs",
3604           "type": "timeseries"
3605         }
3606       ],
3607       "targets": [
3608         {
3609           "datasource": {
3610             "type": "datasource",
3611             "uid": "grafana"
3612           },
3613           "refId": "A"
3614         }
3615       ],
3616       "title": "Producer Performance",
3617       "type": "row"
3618     },
3619     {
3620       "collapsed": true,
3621       "datasource": {
3622         "type": "datasource",
3623         "uid": "grafana"
3624       },
3625       "gridPos": {
3626         "h": 1,
3627         "w": 24,
3628         "x": 0,
3629         "y": 50
3630       },
3631       "id": 68,
3632       "panels": [
3633         {
3634           "datasource": {
3635             "type": "prometheus",
3636             "uid": "PBFA97CFB590B2093"
3637           },
3638           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3639           "fieldConfig": {
3640             "defaults": {
3641               "color": {
3642                 "mode": "palette-classic"
3643               },
3644               "custom": {
3645                 "axisLabel": "",
3646                 "axisPlacement": "auto",
3647                 "barAlignment": 0,
3648                 "drawStyle": "line",
3649                 "fillOpacity": 10,
3650                 "gradientMode": "none",
3651                 "hideFrom": {
3652                   "legend": false,
3653                   "tooltip": false,
3654                   "viz": false
3655                 },
3656                 "lineInterpolation": "linear",
3657                 "lineWidth": 1,
3658                 "pointSize": 5,
3659                 "scaleDistribution": {
3660                   "type": "linear"
3661                 },
3662                 "showPoints": "never",
3663                 "spanNulls": true,
3664                 "stacking": {
3665                   "group": "A",
3666                   "mode": "none"
3667                 },
3668                 "thresholdsStyle": {
3669                   "mode": "off"
3670                 }
3671               },
3672               "links": [],
3673               "mappings": [],
3674               "min": 0,
3675               "thresholds": {
3676                 "mode": "absolute",
3677                 "steps": [
3678                   {
3679                     "color": "green"
3680                   },
3681                   {
3682                     "color": "red",
3683                     "value": 80
3684                   }
3685                 ]
3686               },
3687               "unit": "ms"
3688             },
3689             "overrides": []
3690           },
3691           "gridPos": {
3692             "h": 9,
3693             "w": 12,
3694             "x": 0,
3695             "y": 30
3696           },
3697           "id": 69,
3698           "options": {
3699             "legend": {
3700               "calcs": [
3701                 "mean",
3702                 "lastNotNull",
3703                 "max"
3704               ],
3705               "displayMode": "list",
3706               "placement": "bottom",
3707               "showLegend": true
3708             },
3709             "tooltip": {
3710               "mode": "single"
3711             }
3712           },
3713           "pluginVersion": "8.1.3",
3714           "targets": [
3715             {
3716               "datasource": {
3717                 "type": "prometheus",
3718                 "uid": "PBFA97CFB590B2093"
3719               },
3720               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3721               "hide": false,
3722               "legendFormat": "{{instance}} - {{quantile}}",
3723               "refId": "A"
3724             }
3725           ],
3726           "title": "Consumer - RequestQueueTimeMs",
3727           "type": "timeseries"
3728         },
3729         {
3730           "datasource": {
3731             "type": "prometheus",
3732             "uid": "PBFA97CFB590B2093"
3733           },
3734           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3735           "fieldConfig": {
3736             "defaults": {
3737               "color": {
3738                 "mode": "palette-classic"
3739               },
3740               "custom": {
3741                 "axisLabel": "",
3742                 "axisPlacement": "auto",
3743                 "barAlignment": 0,
3744                 "drawStyle": "line",
3745                 "fillOpacity": 10,
3746                 "gradientMode": "none",
3747                 "hideFrom": {
3748                   "legend": false,
3749                   "tooltip": false,
3750                   "viz": false
3751                 },
3752                 "lineInterpolation": "linear",
3753                 "lineWidth": 1,
3754                 "pointSize": 5,
3755                 "scaleDistribution": {
3756                   "type": "linear"
3757                 },
3758                 "showPoints": "never",
3759                 "spanNulls": true,
3760                 "stacking": {
3761                   "group": "A",
3762                   "mode": "none"
3763                 },
3764                 "thresholdsStyle": {
3765                   "mode": "off"
3766                 }
3767               },
3768               "links": [],
3769               "mappings": [],
3770               "thresholds": {
3771                 "mode": "absolute",
3772                 "steps": [
3773                   {
3774                     "color": "green"
3775                   },
3776                   {
3777                     "color": "red",
3778                     "value": 80
3779                   }
3780                 ]
3781               },
3782               "unit": "ms"
3783             },
3784             "overrides": []
3785           },
3786           "gridPos": {
3787             "h": 9,
3788             "w": 12,
3789             "x": 12,
3790             "y": 30
3791           },
3792           "id": 70,
3793           "options": {
3794             "legend": {
3795               "calcs": [
3796                 "mean",
3797                 "lastNotNull",
3798                 "max"
3799               ],
3800               "displayMode": "list",
3801               "placement": "bottom",
3802               "showLegend": true
3803             },
3804             "tooltip": {
3805               "mode": "single"
3806             }
3807           },
3808           "pluginVersion": "8.1.3",
3809           "targets": [
3810             {
3811               "datasource": {
3812                 "type": "prometheus",
3813                 "uid": "PBFA97CFB590B2093"
3814               },
3815               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3816               "hide": false,
3817               "legendFormat": "{{instance}} - {{quantile}}",
3818               "refId": "A"
3819             }
3820           ],
3821           "title": "Consumer - LocalTimeMs",
3822           "type": "timeseries"
3823         },
3824         {
3825           "datasource": {
3826             "type": "prometheus",
3827             "uid": "PBFA97CFB590B2093"
3828           },
3829           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3830           "fieldConfig": {
3831             "defaults": {
3832               "color": {
3833                 "mode": "palette-classic"
3834               },
3835               "custom": {
3836                 "axisLabel": "",
3837                 "axisPlacement": "auto",
3838                 "barAlignment": 0,
3839                 "drawStyle": "line",
3840                 "fillOpacity": 10,
3841                 "gradientMode": "none",
3842                 "hideFrom": {
3843                   "legend": false,
3844                   "tooltip": false,
3845                   "viz": false
3846                 },
3847                 "lineInterpolation": "linear",
3848                 "lineWidth": 1,
3849                 "pointSize": 5,
3850                 "scaleDistribution": {
3851                   "type": "linear"
3852                 },
3853                 "showPoints": "never",
3854                 "spanNulls": true,
3855                 "stacking": {
3856                   "group": "A",
3857                   "mode": "none"
3858                 },
3859                 "thresholdsStyle": {
3860                   "mode": "off"
3861                 }
3862               },
3863               "links": [],
3864               "mappings": [],
3865               "thresholds": {
3866                 "mode": "absolute",
3867                 "steps": [
3868                   {
3869                     "color": "green"
3870                   },
3871                   {
3872                     "color": "red",
3873                     "value": 80
3874                   }
3875                 ]
3876               },
3877               "unit": "ms"
3878             },
3879             "overrides": []
3880           },
3881           "gridPos": {
3882             "h": 9,
3883             "w": 8,
3884             "x": 0,
3885             "y": 39
3886           },
3887           "id": 71,
3888           "options": {
3889             "legend": {
3890               "calcs": [
3891                 "mean",
3892                 "lastNotNull",
3893                 "max"
3894               ],
3895               "displayMode": "table",
3896               "placement": "bottom",
3897               "showLegend": true
3898             },
3899             "tooltip": {
3900               "mode": "single"
3901             }
3902           },
3903           "pluginVersion": "8.1.3",
3904           "targets": [
3905             {
3906               "datasource": {
3907                 "type": "prometheus",
3908                 "uid": "PBFA97CFB590B2093"
3909               },
3910               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3911               "hide": false,
3912               "legendFormat": "{{instance}} - {{quantile}}",
3913               "refId": "A"
3914             }
3915           ],
3916           "title": "Consumer - RemoteTimeMs",
3917           "type": "timeseries"
3918         },
3919         {
3920           "datasource": {
3921             "type": "prometheus",
3922             "uid": "PBFA97CFB590B2093"
3923           },
3924           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3925           "fieldConfig": {
3926             "defaults": {
3927               "color": {
3928                 "mode": "palette-classic"
3929               },
3930               "custom": {
3931                 "axisLabel": "",
3932                 "axisPlacement": "auto",
3933                 "barAlignment": 0,
3934                 "drawStyle": "line",
3935                 "fillOpacity": 10,
3936                 "gradientMode": "none",
3937                 "hideFrom": {
3938                   "legend": false,
3939                   "tooltip": false,
3940                   "viz": false
3941                 },
3942                 "lineInterpolation": "linear",
3943                 "lineWidth": 1,
3944                 "pointSize": 5,
3945                 "scaleDistribution": {
3946                   "type": "linear"
3947                 },
3948                 "showPoints": "never",
3949                 "spanNulls": true,
3950                 "stacking": {
3951                   "group": "A",
3952                   "mode": "none"
3953                 },
3954                 "thresholdsStyle": {
3955                   "mode": "off"
3956                 }
3957               },
3958               "links": [],
3959               "mappings": [],
3960               "min": 0,
3961               "thresholds": {
3962                 "mode": "absolute",
3963                 "steps": [
3964                   {
3965                     "color": "green"
3966                   },
3967                   {
3968                     "color": "red",
3969                     "value": 80
3970                   }
3971                 ]
3972               },
3973               "unit": "ms"
3974             },
3975             "overrides": []
3976           },
3977           "gridPos": {
3978             "h": 9,
3979             "w": 8,
3980             "x": 8,
3981             "y": 39
3982           },
3983           "id": 72,
3984           "options": {
3985             "legend": {
3986               "calcs": [
3987                 "mean",
3988                 "lastNotNull",
3989                 "max"
3990               ],
3991               "displayMode": "table",
3992               "placement": "bottom",
3993               "showLegend": true
3994             },
3995             "tooltip": {
3996               "mode": "single"
3997             }
3998           },
3999           "pluginVersion": "8.1.3",
4000           "targets": [
4001             {
4002               "datasource": {
4003                 "type": "prometheus",
4004                 "uid": "PBFA97CFB590B2093"
4005               },
4006               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4007               "hide": false,
4008               "legendFormat": "{{instance}} - {{quantile}}",
4009               "refId": "A"
4010             }
4011           ],
4012           "title": "Consumer - ResponseQueueTimeMs",
4013           "type": "timeseries"
4014         },
4015         {
4016           "datasource": {
4017             "type": "prometheus",
4018             "uid": "PBFA97CFB590B2093"
4019           },
4020           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4021           "fieldConfig": {
4022             "defaults": {
4023               "color": {
4024                 "mode": "palette-classic"
4025               },
4026               "custom": {
4027                 "axisLabel": "",
4028                 "axisPlacement": "auto",
4029                 "barAlignment": 0,
4030                 "drawStyle": "line",
4031                 "fillOpacity": 10,
4032                 "gradientMode": "none",
4033                 "hideFrom": {
4034                   "legend": false,
4035                   "tooltip": false,
4036                   "viz": false
4037                 },
4038                 "lineInterpolation": "linear",
4039                 "lineWidth": 1,
4040                 "pointSize": 5,
4041                 "scaleDistribution": {
4042                   "type": "linear"
4043                 },
4044                 "showPoints": "never",
4045                 "spanNulls": true,
4046                 "stacking": {
4047                   "group": "A",
4048                   "mode": "none"
4049                 },
4050                 "thresholdsStyle": {
4051                   "mode": "off"
4052                 }
4053               },
4054               "links": [],
4055               "mappings": [],
4056               "min": 0,
4057               "thresholds": {
4058                 "mode": "absolute",
4059                 "steps": [
4060                   {
4061                     "color": "green"
4062                   },
4063                   {
4064                     "color": "red",
4065                     "value": 80
4066                   }
4067                 ]
4068               },
4069               "unit": "ms"
4070             },
4071             "overrides": []
4072           },
4073           "gridPos": {
4074             "h": 9,
4075             "w": 8,
4076             "x": 16,
4077             "y": 39
4078           },
4079           "id": 73,
4080           "options": {
4081             "legend": {
4082               "calcs": [
4083                 "mean",
4084                 "lastNotNull",
4085                 "max"
4086               ],
4087               "displayMode": "table",
4088               "placement": "bottom",
4089               "showLegend": true
4090             },
4091             "tooltip": {
4092               "mode": "single"
4093             }
4094           },
4095           "pluginVersion": "8.1.3",
4096           "targets": [
4097             {
4098               "datasource": {
4099                 "type": "prometheus",
4100                 "uid": "PBFA97CFB590B2093"
4101               },
4102               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4103               "hide": false,
4104               "legendFormat": "{{instance}} - {{quantile}}",
4105               "refId": "A"
4106             }
4107           ],
4108           "title": "Consumer - ResponseSendTimeMs",
4109           "type": "timeseries"
4110         }
4111       ],
4112       "targets": [
4113         {
4114           "datasource": {
4115             "type": "datasource",
4116             "uid": "grafana"
4117           },
4118           "refId": "A"
4119         }
4120       ],
4121       "title": "Consumer Performance",
4122       "type": "row"
4123     },
4124     {
4125       "collapsed": true,
4126       "datasource": {
4127         "type": "datasource",
4128         "uid": "grafana"
4129       },
4130       "gridPos": {
4131         "h": 1,
4132         "w": 24,
4133         "x": 0,
4134         "y": 51
4135       },
4136       "id": 66,
4137       "panels": [
4138         {
4139           "datasource": {
4140             "type": "prometheus",
4141             "uid": "PBFA97CFB590B2093"
4142           },
4143           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4144           "fieldConfig": {
4145             "defaults": {
4146               "color": {
4147                 "mode": "palette-classic"
4148               },
4149               "custom": {
4150                 "axisLabel": "",
4151                 "axisPlacement": "auto",
4152                 "barAlignment": 0,
4153                 "drawStyle": "line",
4154                 "fillOpacity": 10,
4155                 "gradientMode": "none",
4156                 "hideFrom": {
4157                   "legend": false,
4158                   "tooltip": false,
4159                   "viz": false
4160                 },
4161                 "lineInterpolation": "linear",
4162                 "lineWidth": 1,
4163                 "pointSize": 5,
4164                 "scaleDistribution": {
4165                   "type": "linear"
4166                 },
4167                 "showPoints": "never",
4168                 "spanNulls": true,
4169                 "stacking": {
4170                   "group": "A",
4171                   "mode": "none"
4172                 },
4173                 "thresholdsStyle": {
4174                   "mode": "off"
4175                 }
4176               },
4177               "links": [],
4178               "mappings": [],
4179               "min": 0,
4180               "thresholds": {
4181                 "mode": "absolute",
4182                 "steps": [
4183                   {
4184                     "color": "green"
4185                   },
4186                   {
4187                     "color": "red",
4188                     "value": 80
4189                   }
4190                 ]
4191               },
4192               "unit": "ms"
4193             },
4194             "overrides": []
4195           },
4196           "gridPos": {
4197             "h": 9,
4198             "w": 12,
4199             "x": 0,
4200             "y": 31
4201           },
4202           "id": 74,
4203           "options": {
4204             "legend": {
4205               "calcs": [
4206                 "mean",
4207                 "lastNotNull",
4208                 "max"
4209               ],
4210               "displayMode": "list",
4211               "placement": "bottom",
4212               "showLegend": true
4213             },
4214             "tooltip": {
4215               "mode": "single"
4216             }
4217           },
4218           "pluginVersion": "8.1.3",
4219           "targets": [
4220             {
4221               "datasource": {
4222                 "type": "prometheus",
4223                 "uid": "PBFA97CFB590B2093"
4224               },
4225               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4226               "hide": false,
4227               "legendFormat": "{{instance}} - {{quantile}}",
4228               "refId": "A"
4229             }
4230           ],
4231           "title": "FetchFollower - RequestQueueTimeMs",
4232           "type": "timeseries"
4233         },
4234         {
4235           "datasource": {
4236             "type": "prometheus",
4237             "uid": "PBFA97CFB590B2093"
4238           },
4239           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4240           "fieldConfig": {
4241             "defaults": {
4242               "color": {
4243                 "mode": "palette-classic"
4244               },
4245               "custom": {
4246                 "axisLabel": "",
4247                 "axisPlacement": "auto",
4248                 "barAlignment": 0,
4249                 "drawStyle": "line",
4250                 "fillOpacity": 10,
4251                 "gradientMode": "none",
4252                 "hideFrom": {
4253                   "legend": false,
4254                   "tooltip": false,
4255                   "viz": false
4256                 },
4257                 "lineInterpolation": "linear",
4258                 "lineWidth": 1,
4259                 "pointSize": 5,
4260                 "scaleDistribution": {
4261                   "type": "linear"
4262                 },
4263                 "showPoints": "never",
4264                 "spanNulls": true,
4265                 "stacking": {
4266                   "group": "A",
4267                   "mode": "none"
4268                 },
4269                 "thresholdsStyle": {
4270                   "mode": "off"
4271                 }
4272               },
4273               "links": [],
4274               "mappings": [],
4275               "min": 0,
4276               "thresholds": {
4277                 "mode": "absolute",
4278                 "steps": [
4279                   {
4280                     "color": "green"
4281                   },
4282                   {
4283                     "color": "red",
4284                     "value": 80
4285                   }
4286                 ]
4287               },
4288               "unit": "ms"
4289             },
4290             "overrides": []
4291           },
4292           "gridPos": {
4293             "h": 9,
4294             "w": 12,
4295             "x": 12,
4296             "y": 31
4297           },
4298           "id": 75,
4299           "options": {
4300             "legend": {
4301               "calcs": [
4302                 "mean",
4303                 "lastNotNull",
4304                 "max"
4305               ],
4306               "displayMode": "list",
4307               "placement": "bottom",
4308               "showLegend": true
4309             },
4310             "tooltip": {
4311               "mode": "single"
4312             }
4313           },
4314           "pluginVersion": "8.1.3",
4315           "targets": [
4316             {
4317               "datasource": {
4318                 "type": "prometheus",
4319                 "uid": "PBFA97CFB590B2093"
4320               },
4321               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4322               "hide": false,
4323               "legendFormat": "{{instance}} - {{quantile}}",
4324               "refId": "A"
4325             }
4326           ],
4327           "title": "FetchFollower - LocalTimeMs",
4328           "type": "timeseries"
4329         },
4330         {
4331           "datasource": {
4332             "type": "prometheus",
4333             "uid": "PBFA97CFB590B2093"
4334           },
4335           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4336           "fieldConfig": {
4337             "defaults": {
4338               "color": {
4339                 "mode": "palette-classic"
4340               },
4341               "custom": {
4342                 "axisLabel": "",
4343                 "axisPlacement": "auto",
4344                 "barAlignment": 0,
4345                 "drawStyle": "line",
4346                 "fillOpacity": 10,
4347                 "gradientMode": "none",
4348                 "hideFrom": {
4349                   "legend": false,
4350                   "tooltip": false,
4351                   "viz": false
4352                 },
4353                 "lineInterpolation": "linear",
4354                 "lineWidth": 1,
4355                 "pointSize": 5,
4356                 "scaleDistribution": {
4357                   "type": "linear"
4358                 },
4359                 "showPoints": "never",
4360                 "spanNulls": true,
4361                 "stacking": {
4362                   "group": "A",
4363                   "mode": "none"
4364                 },
4365                 "thresholdsStyle": {
4366                   "mode": "off"
4367                 }
4368               },
4369               "links": [],
4370               "mappings": [],
4371               "thresholds": {
4372                 "mode": "absolute",
4373                 "steps": [
4374                   {
4375                     "color": "green"
4376                   },
4377                   {
4378                     "color": "red",
4379                     "value": 80
4380                   }
4381                 ]
4382               },
4383               "unit": "ms"
4384             },
4385             "overrides": []
4386           },
4387           "gridPos": {
4388             "h": 9,
4389             "w": 8,
4390             "x": 0,
4391             "y": 40
4392           },
4393           "id": 76,
4394           "options": {
4395             "legend": {
4396               "calcs": [
4397                 "mean",
4398                 "lastNotNull",
4399                 "max"
4400               ],
4401               "displayMode": "table",
4402               "placement": "bottom",
4403               "showLegend": true
4404             },
4405             "tooltip": {
4406               "mode": "single"
4407             }
4408           },
4409           "pluginVersion": "8.1.3",
4410           "targets": [
4411             {
4412               "datasource": {
4413                 "type": "prometheus",
4414                 "uid": "PBFA97CFB590B2093"
4415               },
4416               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4417               "hide": false,
4418               "legendFormat": "{{instance}} - {{quantile}}",
4419               "refId": "A"
4420             }
4421           ],
4422           "title": "FetchFollower - RemoteTimeMs",
4423           "type": "timeseries"
4424         },
4425         {
4426           "datasource": {
4427             "type": "prometheus",
4428             "uid": "PBFA97CFB590B2093"
4429           },
4430           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4431           "fieldConfig": {
4432             "defaults": {
4433               "color": {
4434                 "mode": "palette-classic"
4435               },
4436               "custom": {
4437                 "axisLabel": "",
4438                 "axisPlacement": "auto",
4439                 "barAlignment": 0,
4440                 "drawStyle": "line",
4441                 "fillOpacity": 10,
4442                 "gradientMode": "none",
4443                 "hideFrom": {
4444                   "legend": false,
4445                   "tooltip": false,
4446                   "viz": false
4447                 },
4448                 "lineInterpolation": "linear",
4449                 "lineWidth": 1,
4450                 "pointSize": 5,
4451                 "scaleDistribution": {
4452                   "type": "linear"
4453                 },
4454                 "showPoints": "never",
4455                 "spanNulls": true,
4456                 "stacking": {
4457                   "group": "A",
4458                   "mode": "none"
4459                 },
4460                 "thresholdsStyle": {
4461                   "mode": "off"
4462                 }
4463               },
4464               "links": [],
4465               "mappings": [],
4466               "thresholds": {
4467                 "mode": "absolute",
4468                 "steps": [
4469                   {
4470                     "color": "green"
4471                   },
4472                   {
4473                     "color": "red",
4474                     "value": 80
4475                   }
4476                 ]
4477               },
4478               "unit": "ms"
4479             },
4480             "overrides": []
4481           },
4482           "gridPos": {
4483             "h": 9,
4484             "w": 8,
4485             "x": 8,
4486             "y": 40
4487           },
4488           "id": 77,
4489           "options": {
4490             "legend": {
4491               "calcs": [
4492                 "mean",
4493                 "lastNotNull",
4494                 "max"
4495               ],
4496               "displayMode": "table",
4497               "placement": "bottom",
4498               "showLegend": true
4499             },
4500             "tooltip": {
4501               "mode": "single"
4502             }
4503           },
4504           "pluginVersion": "8.1.3",
4505           "targets": [
4506             {
4507               "datasource": {
4508                 "type": "prometheus",
4509                 "uid": "PBFA97CFB590B2093"
4510               },
4511               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4512               "hide": false,
4513               "legendFormat": "{{instance}} - {{quantile}}",
4514               "refId": "A"
4515             }
4516           ],
4517           "title": "FetchFollower - ResponseQueueTimeMs",
4518           "type": "timeseries"
4519         },
4520         {
4521           "datasource": {
4522             "type": "prometheus",
4523             "uid": "PBFA97CFB590B2093"
4524           },
4525           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4526           "fieldConfig": {
4527             "defaults": {
4528               "color": {
4529                 "mode": "palette-classic"
4530               },
4531               "custom": {
4532                 "axisLabel": "",
4533                 "axisPlacement": "auto",
4534                 "barAlignment": 0,
4535                 "drawStyle": "line",
4536                 "fillOpacity": 10,
4537                 "gradientMode": "none",
4538                 "hideFrom": {
4539                   "legend": false,
4540                   "tooltip": false,
4541                   "viz": false
4542                 },
4543                 "lineInterpolation": "linear",
4544                 "lineWidth": 1,
4545                 "pointSize": 5,
4546                 "scaleDistribution": {
4547                   "type": "linear"
4548                 },
4549                 "showPoints": "never",
4550                 "spanNulls": true,
4551                 "stacking": {
4552                   "group": "A",
4553                   "mode": "none"
4554                 },
4555                 "thresholdsStyle": {
4556                   "mode": "off"
4557                 }
4558               },
4559               "links": [],
4560               "mappings": [],
4561               "min": 0,
4562               "thresholds": {
4563                 "mode": "absolute",
4564                 "steps": [
4565                   {
4566                     "color": "green"
4567                   },
4568                   {
4569                     "color": "red",
4570                     "value": 80
4571                   }
4572                 ]
4573               },
4574               "unit": "ms"
4575             },
4576             "overrides": []
4577           },
4578           "gridPos": {
4579             "h": 9,
4580             "w": 8,
4581             "x": 16,
4582             "y": 40
4583           },
4584           "id": 78,
4585           "options": {
4586             "legend": {
4587               "calcs": [
4588                 "mean",
4589                 "lastNotNull",
4590                 "max"
4591               ],
4592               "displayMode": "table",
4593               "placement": "bottom",
4594               "showLegend": true
4595             },
4596             "tooltip": {
4597               "mode": "single"
4598             }
4599           },
4600           "pluginVersion": "8.1.3",
4601           "targets": [
4602             {
4603               "datasource": {
4604                 "type": "prometheus",
4605                 "uid": "PBFA97CFB590B2093"
4606               },
4607               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4608               "hide": false,
4609               "legendFormat": "{{instance}} - {{quantile}}",
4610               "refId": "A"
4611             }
4612           ],
4613           "title": "FetchFollower - ResponseSendTimeMs",
4614           "type": "timeseries"
4615         }
4616       ],
4617       "targets": [
4618         {
4619           "datasource": {
4620             "type": "datasource",
4621             "uid": "grafana"
4622           },
4623           "refId": "A"
4624         }
4625       ],
4626       "title": "Fetch Follower Performance",
4627       "type": "row"
4628     },
4629     {
4630       "collapsed": true,
4631       "datasource": {
4632         "type": "datasource",
4633         "uid": "grafana"
4634       },
4635       "gridPos": {
4636         "h": 1,
4637         "w": 24,
4638         "x": 0,
4639         "y": 52
4640       },
4641       "id": 102,
4642       "panels": [
4643         {
4644           "datasource": {
4645             "type": "prometheus",
4646             "uid": "PBFA97CFB590B2093"
4647           },
4648           "fieldConfig": {
4649             "defaults": {
4650               "color": {
4651                 "mode": "palette-classic"
4652               },
4653               "custom": {
4654                 "axisLabel": "",
4655                 "axisPlacement": "auto",
4656                 "barAlignment": 0,
4657                 "drawStyle": "line",
4658                 "fillOpacity": 10,
4659                 "gradientMode": "none",
4660                 "hideFrom": {
4661                   "legend": false,
4662                   "tooltip": false,
4663                   "viz": false
4664                 },
4665                 "lineInterpolation": "linear",
4666                 "lineWidth": 1,
4667                 "pointSize": 5,
4668                 "scaleDistribution": {
4669                   "type": "linear"
4670                 },
4671                 "showPoints": "never",
4672                 "spanNulls": true,
4673                 "stacking": {
4674                   "group": "A",
4675                   "mode": "none"
4676                 },
4677                 "thresholdsStyle": {
4678                   "mode": "off"
4679                 }
4680               },
4681               "links": [],
4682               "mappings": [],
4683               "thresholds": {
4684                 "mode": "absolute",
4685                 "steps": [
4686                   {
4687                     "color": "green"
4688                   },
4689                   {
4690                     "color": "red",
4691                     "value": 80
4692                   }
4693                 ]
4694               },
4695               "unit": "short"
4696             },
4697             "overrides": []
4698           },
4699           "gridPos": {
4700             "h": 8,
4701             "w": 12,
4702             "x": 0,
4703             "y": 32
4704           },
4705           "id": 98,
4706           "options": {
4707             "legend": {
4708               "calcs": [
4709                 "mean",
4710                 "lastNotNull",
4711                 "max"
4712               ],
4713               "displayMode": "table",
4714               "placement": "bottom",
4715               "showLegend": true
4716             },
4717             "tooltip": {
4718               "mode": "single"
4719             }
4720           },
4721           "pluginVersion": "8.1.3",
4722           "targets": [
4723             {
4724               "datasource": {
4725                 "type": "prometheus",
4726                 "uid": "PBFA97CFB590B2093"
4727               },
4728               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4729               "interval": "",
4730               "legendFormat": "{{listener}}",
4731               "refId": "A"
4732             }
4733           ],
4734           "title": "Connections count per listener",
4735           "type": "timeseries"
4736         },
4737         {
4738           "datasource": {
4739             "type": "prometheus",
4740             "uid": "PBFA97CFB590B2093"
4741           },
4742           "fieldConfig": {
4743             "defaults": {
4744               "color": {
4745                 "mode": "palette-classic"
4746               },
4747               "custom": {
4748                 "axisLabel": "",
4749                 "axisPlacement": "auto",
4750                 "barAlignment": 0,
4751                 "drawStyle": "line",
4752                 "fillOpacity": 10,
4753                 "gradientMode": "none",
4754                 "hideFrom": {
4755                   "legend": false,
4756                   "tooltip": false,
4757                   "viz": false
4758                 },
4759                 "lineInterpolation": "linear",
4760                 "lineWidth": 1,
4761                 "pointSize": 5,
4762                 "scaleDistribution": {
4763                   "type": "linear"
4764                 },
4765                 "showPoints": "never",
4766                 "spanNulls": true,
4767                 "stacking": {
4768                   "group": "A",
4769                   "mode": "none"
4770                 },
4771                 "thresholdsStyle": {
4772                   "mode": "off"
4773                 }
4774               },
4775               "links": [],
4776               "mappings": [],
4777               "thresholds": {
4778                 "mode": "absolute",
4779                 "steps": [
4780                   {
4781                     "color": "green"
4782                   },
4783                   {
4784                     "color": "red",
4785                     "value": 80
4786                   }
4787                 ]
4788               },
4789               "unit": "short"
4790             },
4791             "overrides": []
4792           },
4793           "gridPos": {
4794             "h": 8,
4795             "w": 12,
4796             "x": 12,
4797             "y": 32
4798           },
4799           "id": 100,
4800           "options": {
4801             "legend": {
4802               "calcs": [
4803                 "mean",
4804                 "lastNotNull",
4805                 "max"
4806               ],
4807               "displayMode": "table",
4808               "placement": "bottom",
4809               "showLegend": true
4810             },
4811             "tooltip": {
4812               "mode": "single"
4813             }
4814           },
4815           "pluginVersion": "8.1.3",
4816           "targets": [
4817             {
4818               "datasource": {
4819                 "type": "prometheus",
4820                 "uid": "PBFA97CFB590B2093"
4821               },
4822               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
4823               "interval": "",
4824               "legendFormat": "{{instance}}",
4825               "refId": "A"
4826             }
4827           ],
4828           "title": "Connections count per broker",
4829           "type": "timeseries"
4830         },
4831         {
4832           "datasource": {
4833             "type": "prometheus",
4834             "uid": "PBFA97CFB590B2093"
4835           },
4836           "fieldConfig": {
4837             "defaults": {
4838               "color": {
4839                 "mode": "palette-classic"
4840               },
4841               "custom": {
4842                 "axisLabel": "",
4843                 "axisPlacement": "auto",
4844                 "barAlignment": 0,
4845                 "drawStyle": "line",
4846                 "fillOpacity": 10,
4847                 "gradientMode": "none",
4848                 "hideFrom": {
4849                   "legend": false,
4850                   "tooltip": false,
4851                   "viz": false
4852                 },
4853                 "lineInterpolation": "linear",
4854                 "lineWidth": 1,
4855                 "pointSize": 5,
4856                 "scaleDistribution": {
4857                   "type": "linear"
4858                 },
4859                 "showPoints": "never",
4860                 "spanNulls": true,
4861                 "stacking": {
4862                   "group": "A",
4863                   "mode": "none"
4864                 },
4865                 "thresholdsStyle": {
4866                   "mode": "off"
4867                 }
4868               },
4869               "links": [],
4870               "mappings": [],
4871               "thresholds": {
4872                 "mode": "absolute",
4873                 "steps": [
4874                   {
4875                     "color": "green"
4876                   },
4877                   {
4878                     "color": "red",
4879                     "value": 80
4880                   }
4881                 ]
4882               },
4883               "unit": "short"
4884             },
4885             "overrides": []
4886           },
4887           "gridPos": {
4888             "h": 8,
4889             "w": 12,
4890             "x": 0,
4891             "y": 40
4892           },
4893           "id": 104,
4894           "options": {
4895             "legend": {
4896               "calcs": [
4897                 "mean",
4898                 "lastNotNull",
4899                 "max"
4900               ],
4901               "displayMode": "table",
4902               "placement": "bottom",
4903               "showLegend": true
4904             },
4905             "tooltip": {
4906               "mode": "single"
4907             }
4908           },
4909           "pluginVersion": "8.1.3",
4910           "targets": [
4911             {
4912               "datasource": {
4913                 "type": "prometheus",
4914                 "uid": "PBFA97CFB590B2093"
4915               },
4916               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4917               "interval": "",
4918               "legendFormat": "{{listener}}",
4919               "refId": "A"
4920             }
4921           ],
4922           "title": "Connections creation rate per listener",
4923           "type": "timeseries"
4924         },
4925         {
4926           "datasource": {
4927             "type": "prometheus",
4928             "uid": "PBFA97CFB590B2093"
4929           },
4930           "fieldConfig": {
4931             "defaults": {
4932               "color": {
4933                 "mode": "palette-classic"
4934               },
4935               "custom": {
4936                 "axisLabel": "",
4937                 "axisPlacement": "auto",
4938                 "barAlignment": 0,
4939                 "drawStyle": "line",
4940                 "fillOpacity": 10,
4941                 "gradientMode": "none",
4942                 "hideFrom": {
4943                   "legend": false,
4944                   "tooltip": false,
4945                   "viz": false
4946                 },
4947                 "lineInterpolation": "linear",
4948                 "lineWidth": 1,
4949                 "pointSize": 5,
4950                 "scaleDistribution": {
4951                   "type": "linear"
4952                 },
4953                 "showPoints": "never",
4954                 "spanNulls": true,
4955                 "stacking": {
4956                   "group": "A",
4957                   "mode": "none"
4958                 },
4959                 "thresholdsStyle": {
4960                   "mode": "off"
4961                 }
4962               },
4963               "links": [],
4964               "mappings": [],
4965               "thresholds": {
4966                 "mode": "absolute",
4967                 "steps": [
4968                   {
4969                     "color": "green"
4970                   },
4971                   {
4972                     "color": "red",
4973                     "value": 80
4974                   }
4975                 ]
4976               },
4977               "unit": "short"
4978             },
4979             "overrides": []
4980           },
4981           "gridPos": {
4982             "h": 8,
4983             "w": 12,
4984             "x": 12,
4985             "y": 40
4986           },
4987           "id": 106,
4988           "options": {
4989             "legend": {
4990               "calcs": [
4991                 "mean",
4992                 "lastNotNull",
4993                 "max"
4994               ],
4995               "displayMode": "table",
4996               "placement": "bottom",
4997               "showLegend": true
4998             },
4999             "tooltip": {
5000               "mode": "single"
5001             }
5002           },
5003           "pluginVersion": "8.1.3",
5004           "targets": [
5005             {
5006               "datasource": {
5007                 "type": "prometheus",
5008                 "uid": "PBFA97CFB590B2093"
5009               },
5010               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5011               "interval": "",
5012               "legendFormat": "{{instance}}",
5013               "refId": "A"
5014             }
5015           ],
5016           "title": "Connections creation rate per instance",
5017           "type": "timeseries"
5018         },
5019         {
5020           "datasource": {
5021             "type": "prometheus",
5022             "uid": "PBFA97CFB590B2093"
5023           },
5024           "fieldConfig": {
5025             "defaults": {
5026               "color": {
5027                 "mode": "palette-classic"
5028               },
5029               "custom": {
5030                 "axisLabel": "",
5031                 "axisPlacement": "auto",
5032                 "barAlignment": 0,
5033                 "drawStyle": "line",
5034                 "fillOpacity": 10,
5035                 "gradientMode": "none",
5036                 "hideFrom": {
5037                   "legend": false,
5038                   "tooltip": false,
5039                   "viz": false
5040                 },
5041                 "lineInterpolation": "linear",
5042                 "lineWidth": 1,
5043                 "pointSize": 5,
5044                 "scaleDistribution": {
5045                   "type": "linear"
5046                 },
5047                 "showPoints": "never",
5048                 "spanNulls": true,
5049                 "stacking": {
5050                   "group": "A",
5051                   "mode": "none"
5052                 },
5053                 "thresholdsStyle": {
5054                   "mode": "off"
5055                 }
5056               },
5057               "links": [],
5058               "mappings": [],
5059               "thresholds": {
5060                 "mode": "absolute",
5061                 "steps": [
5062                   {
5063                     "color": "green"
5064                   },
5065                   {
5066                     "color": "red",
5067                     "value": 80
5068                   }
5069                 ]
5070               },
5071               "unit": "short"
5072             },
5073             "overrides": []
5074           },
5075           "gridPos": {
5076             "h": 8,
5077             "w": 12,
5078             "x": 0,
5079             "y": 48
5080           },
5081           "id": 108,
5082           "options": {
5083             "legend": {
5084               "calcs": [
5085                 "mean",
5086                 "lastNotNull",
5087                 "max"
5088               ],
5089               "displayMode": "table",
5090               "placement": "bottom",
5091               "showLegend": true
5092             },
5093             "tooltip": {
5094               "mode": "single"
5095             }
5096           },
5097           "pluginVersion": "8.1.3",
5098           "targets": [
5099             {
5100               "datasource": {
5101                 "type": "prometheus",
5102                 "uid": "PBFA97CFB590B2093"
5103               },
5104               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5105               "interval": "",
5106               "legendFormat": "{{listener}}",
5107               "refId": "A"
5108             }
5109           ],
5110           "title": "Connections close rate per listener",
5111           "type": "timeseries"
5112         },
5113         {
5114           "datasource": {
5115             "type": "prometheus",
5116             "uid": "PBFA97CFB590B2093"
5117           },
5118           "fieldConfig": {
5119             "defaults": {
5120               "color": {
5121                 "mode": "palette-classic"
5122               },
5123               "custom": {
5124                 "axisLabel": "",
5125                 "axisPlacement": "auto",
5126                 "barAlignment": 0,
5127                 "drawStyle": "line",
5128                 "fillOpacity": 10,
5129                 "gradientMode": "none",
5130                 "hideFrom": {
5131                   "legend": false,
5132                   "tooltip": false,
5133                   "viz": false
5134                 },
5135                 "lineInterpolation": "linear",
5136                 "lineWidth": 1,
5137                 "pointSize": 5,
5138                 "scaleDistribution": {
5139                   "type": "linear"
5140                 },
5141                 "showPoints": "never",
5142                 "spanNulls": true,
5143                 "stacking": {
5144                   "group": "A",
5145                   "mode": "none"
5146                 },
5147                 "thresholdsStyle": {
5148                   "mode": "off"
5149                 }
5150               },
5151               "links": [],
5152               "mappings": [],
5153               "thresholds": {
5154                 "mode": "absolute",
5155                 "steps": [
5156                   {
5157                     "color": "green"
5158                   },
5159                   {
5160                     "color": "red",
5161                     "value": 80
5162                   }
5163                 ]
5164               },
5165               "unit": "short"
5166             },
5167             "overrides": []
5168           },
5169           "gridPos": {
5170             "h": 8,
5171             "w": 12,
5172             "x": 12,
5173             "y": 48
5174           },
5175           "id": 110,
5176           "options": {
5177             "legend": {
5178               "calcs": [
5179                 "mean",
5180                 "lastNotNull",
5181                 "max"
5182               ],
5183               "displayMode": "table",
5184               "placement": "bottom",
5185               "showLegend": true
5186             },
5187             "tooltip": {
5188               "mode": "single"
5189             }
5190           },
5191           "pluginVersion": "8.1.3",
5192           "targets": [
5193             {
5194               "datasource": {
5195                 "type": "prometheus",
5196                 "uid": "PBFA97CFB590B2093"
5197               },
5198               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5199               "interval": "",
5200               "legendFormat": "{{instance}}",
5201               "refId": "A"
5202             }
5203           ],
5204           "title": "Connections close rate per instance",
5205           "type": "timeseries"
5206         },
5207         {
5208           "datasource": {
5209             "type": "prometheus",
5210             "uid": "PBFA97CFB590B2093"
5211           },
5212           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5213           "fieldConfig": {
5214             "defaults": {
5215               "color": {
5216                 "mode": "palette-classic"
5217               },
5218               "custom": {
5219                 "axisLabel": "",
5220                 "axisPlacement": "auto",
5221                 "barAlignment": 0,
5222                 "drawStyle": "line",
5223                 "fillOpacity": 10,
5224                 "gradientMode": "none",
5225                 "hideFrom": {
5226                   "legend": false,
5227                   "tooltip": false,
5228                   "viz": false
5229                 },
5230                 "lineInterpolation": "linear",
5231                 "lineWidth": 1,
5232                 "pointSize": 5,
5233                 "scaleDistribution": {
5234                   "type": "linear"
5235                 },
5236                 "showPoints": "never",
5237                 "spanNulls": true,
5238                 "stacking": {
5239                   "group": "A",
5240                   "mode": "none"
5241                 },
5242                 "thresholdsStyle": {
5243                   "mode": "off"
5244                 }
5245               },
5246               "links": [],
5247               "mappings": [],
5248               "min": 0,
5249               "thresholds": {
5250                 "mode": "absolute",
5251                 "steps": [
5252                   {
5253                     "color": "green"
5254                   },
5255                   {
5256                     "color": "red",
5257                     "value": 80
5258                   }
5259                 ]
5260               },
5261               "unit": "percent"
5262             },
5263             "overrides": []
5264           },
5265           "gridPos": {
5266             "h": 8,
5267             "w": 12,
5268             "x": 0,
5269             "y": 56
5270           },
5271           "id": 112,
5272           "options": {
5273             "legend": {
5274               "calcs": [
5275                 "mean",
5276                 "lastNotNull",
5277                 "max"
5278               ],
5279               "displayMode": "table",
5280               "placement": "bottom",
5281               "showLegend": true
5282             },
5283             "tooltip": {
5284               "mode": "single"
5285             }
5286           },
5287           "pluginVersion": "8.1.3",
5288           "targets": [
5289             {
5290               "datasource": {
5291                 "type": "prometheus",
5292                 "uid": "PBFA97CFB590B2093"
5293               },
5294               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5295               "interval": "",
5296               "legendFormat": "{{instance}} - {{listener}}",
5297               "refId": "A"
5298             }
5299           ],
5300           "title": "Acceptor Blocked Percentage",
5301           "type": "timeseries"
5302         },
5303         {
5304           "datasource": {
5305             "type": "prometheus",
5306             "uid": "PBFA97CFB590B2093"
5307           },
5308           "fieldConfig": {
5309             "defaults": {
5310               "color": {
5311                 "mode": "palette-classic"
5312               },
5313               "custom": {
5314                 "axisLabel": "",
5315                 "axisPlacement": "auto",
5316                 "barAlignment": 0,
5317                 "drawStyle": "line",
5318                 "fillOpacity": 10,
5319                 "gradientMode": "none",
5320                 "hideFrom": {
5321                   "legend": false,
5322                   "tooltip": false,
5323                   "viz": false
5324                 },
5325                 "lineInterpolation": "linear",
5326                 "lineWidth": 1,
5327                 "pointSize": 5,
5328                 "scaleDistribution": {
5329                   "type": "linear"
5330                 },
5331                 "showPoints": "never",
5332                 "spanNulls": true,
5333                 "stacking": {
5334                   "group": "A",
5335                   "mode": "none"
5336                 },
5337                 "thresholdsStyle": {
5338                   "mode": "off"
5339                 }
5340               },
5341               "links": [],
5342               "mappings": [],
5343               "thresholds": {
5344                 "mode": "absolute",
5345                 "steps": [
5346                   {
5347                     "color": "green"
5348                   },
5349                   {
5350                     "color": "red",
5351                     "value": 80
5352                   }
5353                 ]
5354               },
5355               "unit": "short"
5356             },
5357             "overrides": []
5358           },
5359           "gridPos": {
5360             "h": 8,
5361             "w": 12,
5362             "x": 12,
5363             "y": 56
5364           },
5365           "id": 114,
5366           "options": {
5367             "legend": {
5368               "calcs": [
5369                 "mean",
5370                 "lastNotNull",
5371                 "max"
5372               ],
5373               "displayMode": "table",
5374               "placement": "bottom",
5375               "showLegend": true
5376             },
5377             "tooltip": {
5378               "mode": "single"
5379             }
5380           },
5381           "pluginVersion": "8.1.3",
5382           "targets": [
5383             {
5384               "datasource": {
5385                 "type": "prometheus",
5386                 "uid": "PBFA97CFB590B2093"
5387               },
5388               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5389               "interval": "",
5390               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5391               "refId": "A"
5392             }
5393           ],
5394           "title": "Connections per client version",
5395           "type": "timeseries"
5396         }
5397       ],
5398       "targets": [
5399         {
5400           "datasource": {
5401             "type": "datasource",
5402             "uid": "grafana"
5403           },
5404           "refId": "A"
5405         }
5406       ],
5407       "title": "Connections",
5408       "type": "row"
5409     },
5410     {
5411       "collapsed": true,
5412       "datasource": {
5413         "type": "datasource",
5414         "uid": "grafana"
5415       },
5416       "gridPos": {
5417         "h": 1,
5418         "w": 24,
5419         "x": 0,
5420         "y": 53
5421       },
5422       "id": 120,
5423       "panels": [
5424         {
5425           "datasource": {
5426             "type": "prometheus",
5427             "uid": "PBFA97CFB590B2093"
5428           },
5429           "description": "Number of consumer groups per group coordinator",
5430           "fieldConfig": {
5431             "defaults": {
5432               "color": {
5433                 "mode": "palette-classic"
5434               },
5435               "custom": {
5436                 "axisLabel": "",
5437                 "axisPlacement": "auto",
5438                 "barAlignment": 0,
5439                 "drawStyle": "line",
5440                 "fillOpacity": 10,
5441                 "gradientMode": "none",
5442                 "hideFrom": {
5443                   "legend": false,
5444                   "tooltip": false,
5445                   "viz": false
5446                 },
5447                 "lineInterpolation": "linear",
5448                 "lineWidth": 1,
5449                 "pointSize": 5,
5450                 "scaleDistribution": {
5451                   "type": "linear"
5452                 },
5453                 "showPoints": "never",
5454                 "spanNulls": true,
5455                 "stacking": {
5456                   "group": "A",
5457                   "mode": "none"
5458                 },
5459                 "thresholdsStyle": {
5460                   "mode": "off"
5461                 }
5462               },
5463               "links": [],
5464               "mappings": [],
5465               "thresholds": {
5466                 "mode": "absolute",
5467                 "steps": [
5468                   {
5469                     "color": "green"
5470                   },
5471                   {
5472                     "color": "red",
5473                     "value": 80
5474                   }
5475                 ]
5476               },
5477               "unit": "short"
5478             },
5479             "overrides": []
5480           },
5481           "gridPos": {
5482             "h": 8,
5483             "w": 12,
5484             "x": 0,
5485             "y": 33
5486           },
5487           "id": 116,
5488           "options": {
5489             "legend": {
5490               "calcs": [
5491                 "mean",
5492                 "lastNotNull",
5493                 "max"
5494               ],
5495               "displayMode": "table",
5496               "placement": "bottom",
5497               "showLegend": true
5498             },
5499             "tooltip": {
5500               "mode": "single"
5501             }
5502           },
5503           "pluginVersion": "8.1.3",
5504           "targets": [
5505             {
5506               "datasource": {
5507                 "type": "prometheus",
5508                 "uid": "PBFA97CFB590B2093"
5509               },
5510               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5511               "instant": false,
5512               "interval": "",
5513               "legendFormat": "{{instance}}",
5514               "refId": "A"
5515             }
5516           ],
5517           "title": "Consumer groups number per coordinator",
5518           "type": "timeseries"
5519         },
5520         {
5521           "datasource": {
5522             "type": "prometheus",
5523             "uid": "PBFA97CFB590B2093"
5524           },
5525           "description": "Number of consumer group per state",
5526           "fieldConfig": {
5527             "defaults": {
5528               "color": {
5529                 "mode": "palette-classic"
5530               },
5531               "custom": {
5532                 "axisLabel": "",
5533                 "axisPlacement": "auto",
5534                 "barAlignment": 0,
5535                 "drawStyle": "line",
5536                 "fillOpacity": 10,
5537                 "gradientMode": "none",
5538                 "hideFrom": {
5539                   "legend": false,
5540                   "tooltip": false,
5541                   "viz": false
5542                 },
5543                 "lineInterpolation": "linear",
5544                 "lineWidth": 1,
5545                 "pointSize": 5,
5546                 "scaleDistribution": {
5547                   "type": "linear"
5548                 },
5549                 "showPoints": "never",
5550                 "spanNulls": true,
5551                 "stacking": {
5552                   "group": "A",
5553                   "mode": "none"
5554                 },
5555                 "thresholdsStyle": {
5556                   "mode": "off"
5557                 }
5558               },
5559               "links": [],
5560               "mappings": [],
5561               "thresholds": {
5562                 "mode": "absolute",
5563                 "steps": [
5564                   {
5565                     "color": "green"
5566                   },
5567                   {
5568                     "color": "red",
5569                     "value": 80
5570                   }
5571                 ]
5572               },
5573               "unit": "short"
5574             },
5575             "overrides": []
5576           },
5577           "gridPos": {
5578             "h": 8,
5579             "w": 12,
5580             "x": 12,
5581             "y": 33
5582           },
5583           "id": 118,
5584           "options": {
5585             "legend": {
5586               "calcs": [
5587                 "mean",
5588                 "lastNotNull",
5589                 "max"
5590               ],
5591               "displayMode": "table",
5592               "placement": "bottom",
5593               "showLegend": true
5594             },
5595             "tooltip": {
5596               "mode": "single"
5597             }
5598           },
5599           "pluginVersion": "8.1.3",
5600           "targets": [
5601             {
5602               "datasource": {
5603                 "type": "prometheus",
5604                 "uid": "PBFA97CFB590B2093"
5605               },
5606               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5607               "instant": false,
5608               "interval": "",
5609               "legendFormat": "stable",
5610               "refId": "A"
5611             },
5612             {
5613               "datasource": {
5614                 "type": "prometheus",
5615                 "uid": "PBFA97CFB590B2093"
5616               },
5617               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5618               "interval": "",
5619               "legendFormat": "preparing-rebalance",
5620               "refId": "B"
5621             },
5622             {
5623               "datasource": {
5624                 "type": "prometheus",
5625                 "uid": "PBFA97CFB590B2093"
5626               },
5627               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5628               "interval": "",
5629               "legendFormat": "dead",
5630               "refId": "C"
5631             },
5632             {
5633               "datasource": {
5634                 "type": "prometheus",
5635                 "uid": "PBFA97CFB590B2093"
5636               },
5637               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5638               "interval": "",
5639               "legendFormat": "completing-rebalance",
5640               "refId": "D"
5641             },
5642             {
5643               "datasource": {
5644                 "type": "prometheus",
5645                 "uid": "PBFA97CFB590B2093"
5646               },
5647               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5648               "interval": "",
5649               "legendFormat": "empty",
5650               "refId": "E"
5651             }
5652           ],
5653           "title": "Nb consumer groups per state",
5654           "type": "timeseries"
5655         }
5656       ],
5657       "targets": [
5658         {
5659           "datasource": {
5660             "type": "datasource",
5661             "uid": "grafana"
5662           },
5663           "refId": "A"
5664         }
5665       ],
5666       "title": "Group Coordinator",
5667       "type": "row"
5668     },
5669     {
5670       "collapsed": true,
5671       "datasource": {
5672         "type": "datasource",
5673         "uid": "grafana"
5674       },
5675       "gridPos": {
5676         "h": 1,
5677         "w": 24,
5678         "x": 0,
5679         "y": 54
5680       },
5681       "id": 46,
5682       "panels": [
5683         {
5684           "datasource": {
5685             "type": "prometheus",
5686             "uid": "PBFA97CFB590B2093"
5687           },
5688           "description": "The number of messages produced converted to match the log.message.format.version.",
5689           "fieldConfig": {
5690             "defaults": {
5691               "color": {
5692                 "mode": "palette-classic"
5693               },
5694               "custom": {
5695                 "axisLabel": "",
5696                 "axisPlacement": "auto",
5697                 "barAlignment": 0,
5698                 "drawStyle": "line",
5699                 "fillOpacity": 10,
5700                 "gradientMode": "none",
5701                 "hideFrom": {
5702                   "legend": false,
5703                   "tooltip": false,
5704                   "viz": false
5705                 },
5706                 "lineInterpolation": "linear",
5707                 "lineWidth": 1,
5708                 "pointSize": 5,
5709                 "scaleDistribution": {
5710                   "type": "linear"
5711                 },
5712                 "showPoints": "never",
5713                 "spanNulls": true,
5714                 "stacking": {
5715                   "group": "A",
5716                   "mode": "none"
5717                 },
5718                 "thresholdsStyle": {
5719                   "mode": "off"
5720                 }
5721               },
5722               "links": [],
5723               "mappings": [],
5724               "min": 0,
5725               "thresholds": {
5726                 "mode": "absolute",
5727                 "steps": [
5728                   {
5729                     "color": "green"
5730                   },
5731                   {
5732                     "color": "red",
5733                     "value": 80
5734                   }
5735                 ]
5736               },
5737               "unit": "ops"
5738             },
5739             "overrides": []
5740           },
5741           "gridPos": {
5742             "h": 8,
5743             "w": 8,
5744             "x": 0,
5745             "y": 34
5746           },
5747           "id": 48,
5748           "options": {
5749             "legend": {
5750               "calcs": [
5751                 "mean",
5752                 "lastNotNull",
5753                 "max"
5754               ],
5755               "displayMode": "list",
5756               "placement": "bottom",
5757               "showLegend": true
5758             },
5759             "tooltip": {
5760               "mode": "single"
5761             }
5762           },
5763           "pluginVersion": "8.1.3",
5764           "targets": [
5765             {
5766               "datasource": {
5767                 "type": "prometheus",
5768                 "uid": "PBFA97CFB590B2093"
5769               },
5770               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5771               "hide": false,
5772               "interval": "",
5773               "legendFormat": "{{topic}}",
5774               "refId": "A"
5775             }
5776           ],
5777           "title": "Number of produced message conversion",
5778           "type": "timeseries"
5779         },
5780         {
5781           "datasource": {
5782             "type": "prometheus",
5783             "uid": "PBFA97CFB590B2093"
5784           },
5785           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5786           "fieldConfig": {
5787             "defaults": {
5788               "color": {
5789                 "mode": "palette-classic"
5790               },
5791               "custom": {
5792                 "axisLabel": "",
5793                 "axisPlacement": "auto",
5794                 "barAlignment": 0,
5795                 "drawStyle": "line",
5796                 "fillOpacity": 10,
5797                 "gradientMode": "none",
5798                 "hideFrom": {
5799                   "legend": false,
5800                   "tooltip": false,
5801                   "viz": false
5802                 },
5803                 "lineInterpolation": "linear",
5804                 "lineWidth": 1,
5805                 "pointSize": 5,
5806                 "scaleDistribution": {
5807                   "type": "linear"
5808                 },
5809                 "showPoints": "never",
5810                 "spanNulls": true,
5811                 "stacking": {
5812                   "group": "A",
5813                   "mode": "none"
5814                 },
5815                 "thresholdsStyle": {
5816                   "mode": "off"
5817                 }
5818               },
5819               "links": [],
5820               "mappings": [],
5821               "min": 0,
5822               "thresholds": {
5823                 "mode": "absolute",
5824                 "steps": [
5825                   {
5826                     "color": "green"
5827                   },
5828                   {
5829                     "color": "red",
5830                     "value": 80
5831                   }
5832                 ]
5833               },
5834               "unit": "ops"
5835             },
5836             "overrides": []
5837           },
5838           "gridPos": {
5839             "h": 8,
5840             "w": 8,
5841             "x": 8,
5842             "y": 34
5843           },
5844           "id": 51,
5845           "options": {
5846             "legend": {
5847               "calcs": [
5848                 "mean",
5849                 "lastNotNull",
5850                 "max"
5851               ],
5852               "displayMode": "list",
5853               "placement": "bottom",
5854               "showLegend": true
5855             },
5856             "tooltip": {
5857               "mode": "single"
5858             }
5859           },
5860           "pluginVersion": "8.1.3",
5861           "targets": [
5862             {
5863               "datasource": {
5864                 "type": "prometheus",
5865                 "uid": "PBFA97CFB590B2093"
5866               },
5867               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5868               "hide": false,
5869               "interval": "",
5870               "legendFormat": "{{topic}}",
5871               "refId": "A"
5872             }
5873           ],
5874           "title": "Number of consumed message conversion",
5875           "type": "timeseries"
5876         },
5877         {
5878           "datasource": {
5879             "type": "datasource",
5880             "uid": "grafana"
5881           },
5882           "description": "Number of connection per client version",
5883           "fieldConfig": {
5884             "defaults": {
5885               "color": {
5886                 "mode": "palette-classic"
5887               },
5888               "custom": {
5889                 "hideFrom": {
5890                   "legend": false,
5891                   "tooltip": false,
5892                   "viz": false
5893                 }
5894               },
5895               "decimals": 0,
5896               "mappings": [],
5897               "unit": "short"
5898             },
5899             "overrides": []
5900           },
5901           "gridPos": {
5902             "h": 8,
5903             "w": 8,
5904             "x": 16,
5905             "y": 34
5906           },
5907           "id": 96,
5908           "links": [],
5909           "options": {
5910             "legend": {
5911               "calcs": [],
5912               "displayMode": "table",
5913               "placement": "bottom",
5914               "showLegend": true,
5915               "values": [
5916                 "value"
5917               ]
5918             },
5919             "pieType": "pie",
5920             "reduceOptions": {
5921               "calcs": [
5922                 "lastNotNull"
5923               ],
5924               "fields": "",
5925               "values": false
5926             },
5927             "tooltip": {
5928               "mode": "single"
5929             }
5930           },
5931           "targets": [
5932             {
5933               "datasource": {
5934                 "type": "datasource",
5935                 "uid": "grafana"
5936               },
5937               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
5938               "interval": "",
5939               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
5940               "refId": "A"
5941             }
5942           ],
5943           "title": "Client version repartition",
5944           "type": "piechart"
5945         }
5946       ],
5947       "targets": [
5948         {
5949           "datasource": {
5950             "type": "datasource",
5951             "uid": "grafana"
5952           },
5953           "refId": "A"
5954         }
5955       ],
5956       "title": "Message Conversion",
5957       "type": "row"
5958     }
5959   ],
5960   "refresh": "5s",
5961   "schemaVersion": 38,
5962   "style": "dark",
5963   "tags": [],
5964   "templating": {
5965     "list": [
5966       {
5967         "current": {
5968           "selected": false,
5969           "text": "dev",
5970           "value": "dev"
5971         },
5972         "datasource": {
5973           "type": "prometheus",
5974           "uid": "PBFA97CFB590B2093"
5975         },
5976         "definition": "label_values(env)",
5977         "hide": 0,
5978         "includeAll": false,
5979         "label": "Environment",
5980         "multi": false,
5981         "name": "env",
5982         "options": [],
5983         "query": {
5984           "query": "label_values(env)",
5985           "refId": "Prometheus-env-Variable-Query"
5986         },
5987         "refresh": 1,
5988         "regex": "",
5989         "skipUrlSync": false,
5990         "sort": 0,
5991         "tagValuesQuery": "",
5992         "tagsQuery": "",
5993         "type": "query",
5994         "useTags": false
5995       },
5996       {
5997         "current": {
5998           "selected": true,
5999           "text": [
6000             "All"
6001           ],
6002           "value": [
6003             "$__all"
6004           ]
6005         },
6006         "datasource": {
6007           "type": "prometheus",
6008           "uid": "PBFA97CFB590B2093"
6009         },
6010         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6011         "hide": 0,
6012         "includeAll": true,
6013         "label": "Instance",
6014         "multi": true,
6015         "name": "instance",
6016         "options": [],
6017         "query": {
6018           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6019           "refId": "Prometheus-instance-Variable-Query"
6020         },
6021         "refresh": 1,
6022         "regex": "",
6023         "skipUrlSync": false,
6024         "sort": 0,
6025         "tagValuesQuery": "",
6026         "tagsQuery": "",
6027         "type": "query",
6028         "useTags": false
6029       },
6030       {
6031         "allValue": "",
6032         "current": {
6033           "selected": true,
6034           "text": [
6035             "0.95"
6036           ],
6037           "value": [
6038             "0.95"
6039           ]
6040         },
6041         "datasource": {
6042           "type": "prometheus",
6043           "uid": "PBFA97CFB590B2093"
6044         },
6045         "definition": "label_values(quantile)",
6046         "hide": 0,
6047         "includeAll": true,
6048         "label": "Percentile",
6049         "multi": true,
6050         "name": "percentile",
6051         "options": [],
6052         "query": {
6053           "query": "label_values(quantile)",
6054           "refId": "Prometheus-percentile-Variable-Query"
6055         },
6056         "refresh": 1,
6057         "regex": "",
6058         "skipUrlSync": false,
6059         "sort": 0,
6060         "tagValuesQuery": "",
6061         "tagsQuery": "",
6062         "type": "query",
6063         "useTags": false
6064       }
6065     ]
6066   },
6067   "time": {
6068     "from": "now-12h",
6069     "to": "now"
6070   },
6071   "timepicker": {
6072     "refresh_intervals": [
6073       "5s",
6074       "10s",
6075       "30s",
6076       "1m",
6077       "5m",
6078       "15m",
6079       "30m",
6080       "1h",
6081       "2h",
6082       "1d"
6083     ],
6084     "time_options": [
6085       "5m",
6086       "15m",
6087       "1h",
6088       "6h",
6089       "12h",
6090       "24h",
6091       "2d",
6092       "7d",
6093       "30d"
6094     ]
6095   },
6096   "timezone": "browser",
6097   "title": "Kafka cluster",
6098   "uid": "qu-QZdfZz",
6099   "version": 1,
6100   "weekStart": ""
6101 }