juplo.de Git - demos/kafka/monitoring/blob - kafka-overview.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": "-- Grafana --",
   7         "enable": true,
   8         "hide": true,
   9         "iconColor": "rgba(0, 211, 255, 1)",
  10         "name": "Annotations & Alerts",
  11         "target": {
  12           "limit": 100,
  13           "matchAny": false,
  14           "tags": [],
  15           "type": "dashboard"
  16         },
  17         "type": "dashboard"
  18       }
  19     ]
  20   },
  21   "description": "Kafka resource usage and throughput",
  22   "editable": true,
  23   "gnetId": 721,
  24   "graphTooltip": 0,
  25   "id": 6,
  26   "iteration": 1647427255896,
  27   "links": [],
  28   "panels": [
  29     {
  30       "collapsed": false,
  31       "datasource": null,
  32       "gridPos": {
  33         "h": 1,
  34         "w": 24,
  35         "x": 0,
  36         "y": 0
  37       },
  38       "id": 42,
  39       "panels": [],
  40       "title": "Healthcheck",
  41       "type": "row"
  42     },
  43     {
  44       "cacheTimeout": null,
  45       "datasource": "Prometheus",
  46       "description": "Number of active controllers in the cluster.",
  47       "fieldConfig": {
  48         "defaults": {
  49           "mappings": [
  50             {
  51               "options": {
  52                 "match": "null",
  53                 "result": {
  54                   "text": "N/A"
  55                 }
  56               },
  57               "type": "special"
  58             }
  59           ],
  60           "thresholds": {
  61             "mode": "absolute",
  62             "steps": [
  63               {
  64                 "color": "#299c46",
  65                 "value": null
  66               },
  67               {
  68                 "color": "#e5ac0e",
  69                 "value": 2
  70               },
  71               {
  72                 "color": "#bf1b00"
  73               }
  74             ]
  75           },
  76           "unit": "none"
  77         },
  78         "overrides": []
  79       },
  80       "gridPos": {
  81         "h": 4,
  82         "w": 4,
  83         "x": 0,
  84         "y": 1
  85       },
  86       "id": 12,
  87       "interval": null,
  88       "links": [],
  89       "maxDataPoints": 100,
  90       "options": {
  91         "colorMode": "value",
  92         "graphMode": "none",
  93         "justifyMode": "auto",
  94         "orientation": "vertical",
  95         "reduceOptions": {
  96           "calcs": [
  97             "lastNotNull"
  98           ],
  99           "fields": "",
 100           "values": false
 101         },
 102         "text": {},
 103         "textMode": "value_and_name"
 104       },
 105       "pluginVersion": "8.1.3",
 106       "targets": [
 107         {
 108           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 109           "format": "time_series",
 110           "instant": true,
 111           "interval": "",
 112           "intervalFactor": 1,
 113           "legendFormat": "{{instance}}",
 114           "refId": "A"
 115         }
 116       ],
 117       "title": "Active Controllers",
 118       "type": "stat"
 119     },
 120     {
 121       "cacheTimeout": null,
 122       "datasource": "Prometheus",
 123       "description": "Number of Brokers Online",
 124       "fieldConfig": {
 125         "defaults": {
 126           "color": {
 127             "mode": "thresholds"
 128           },
 129           "mappings": [
 130             {
 131               "options": {
 132                 "match": "null",
 133                 "result": {
 134                   "text": "N/A"
 135                 }
 136               },
 137               "type": "special"
 138             }
 139           ],
 140           "thresholds": {
 141             "mode": "absolute",
 142             "steps": [
 143               {
 144                 "color": "#d44a3a",
 145                 "value": null
 146               },
 147               {
 148                 "color": "rgba(237, 129, 40, 0.89)",
 149                 "value": 0
 150               },
 151               {
 152                 "color": "#299c46",
 153                 "value": 2
 154               }
 155             ]
 156           },
 157           "unit": "none"
 158         },
 159         "overrides": []
 160       },
 161       "gridPos": {
 162         "h": 4,
 163         "w": 4,
 164         "x": 4,
 165         "y": 1
 166       },
 167       "id": 14,
 168       "interval": null,
 169       "links": [],
 170       "maxDataPoints": 100,
 171       "options": {
 172         "colorMode": "value",
 173         "graphMode": "none",
 174         "justifyMode": "auto",
 175         "orientation": "horizontal",
 176         "reduceOptions": {
 177           "calcs": [
 178             "lastNotNull"
 179           ],
 180           "fields": "",
 181           "values": false
 182         },
 183         "text": {},
 184         "textMode": "auto"
 185       },
 186       "pluginVersion": "8.1.3",
 187       "repeat": null,
 188       "repeatDirection": "h",
 189       "targets": [
 190         {
 191           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 192           "format": "time_series",
 193           "instant": true,
 194           "interval": "",
 195           "intervalFactor": 1,
 196           "legendFormat": "",
 197           "refId": "A"
 198         }
 199       ],
 200       "title": "Brokers Online",
 201       "type": "stat"
 202     },
 203     {
 204       "cacheTimeout": null,
 205       "datasource": "Prometheus",
 206       "description": "Partitions that are online",
 207       "fieldConfig": {
 208         "defaults": {
 209           "color": {
 210             "mode": "thresholds"
 211           },
 212           "mappings": [
 213             {
 214               "options": {
 215                 "match": "null",
 216                 "result": {
 217                   "text": "N/A"
 218                 }
 219               },
 220               "type": "special"
 221             }
 222           ],
 223           "thresholds": {
 224             "mode": "absolute",
 225             "steps": [
 226               {
 227                 "color": "#d44a3a",
 228                 "value": null
 229               },
 230               {
 231                 "color": "rgba(237, 129, 40, 0.89)",
 232                 "value": 0
 233               },
 234               {
 235                 "color": "#299c46",
 236                 "value": 0
 237               }
 238             ]
 239           },
 240           "unit": "none"
 241         },
 242         "overrides": []
 243       },
 244       "gridPos": {
 245         "h": 4,
 246         "w": 4,
 247         "x": 8,
 248         "y": 1
 249       },
 250       "id": 18,
 251       "interval": null,
 252       "links": [],
 253       "maxDataPoints": 100,
 254       "options": {
 255         "colorMode": "value",
 256         "graphMode": "none",
 257         "justifyMode": "auto",
 258         "orientation": "horizontal",
 259         "reduceOptions": {
 260           "calcs": [
 261             "lastNotNull"
 262           ],
 263           "fields": "",
 264           "values": false
 265         },
 266         "text": {},
 267         "textMode": "auto"
 268       },
 269       "pluginVersion": "8.1.3",
 270       "targets": [
 271         {
 272           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 273           "format": "time_series",
 274           "instant": true,
 275           "interval": "",
 276           "intervalFactor": 1,
 277           "legendFormat": "",
 278           "refId": "A"
 279         }
 280       ],
 281       "title": "Online Partitions",
 282       "type": "stat"
 283     },
 284     {
 285       "cacheTimeout": null,
 286       "datasource": "Prometheus",
 287       "description": "",
 288       "fieldConfig": {
 289         "defaults": {
 290           "color": {
 291             "mode": "thresholds"
 292           },
 293           "mappings": [
 294             {
 295               "options": {
 296                 "match": "null",
 297                 "result": {
 298                   "text": "N/A"
 299                 }
 300               },
 301               "type": "special"
 302             }
 303           ],
 304           "thresholds": {
 305             "mode": "absolute",
 306             "steps": [
 307               {
 308                 "color": "#299c46",
 309                 "value": null
 310               },
 311               {
 312                 "color": "rgba(237, 129, 40, 0.89)",
 313                 "value": 2
 314               },
 315               {
 316                 "color": "#d44a3a"
 317               }
 318             ]
 319           },
 320           "unit": "none"
 321         },
 322         "overrides": []
 323       },
 324       "gridPos": {
 325         "h": 4,
 326         "w": 4,
 327         "x": 12,
 328         "y": 1
 329       },
 330       "id": 33,
 331       "interval": null,
 332       "links": [],
 333       "maxDataPoints": 100,
 334       "options": {
 335         "colorMode": "value",
 336         "graphMode": "none",
 337         "justifyMode": "auto",
 338         "orientation": "horizontal",
 339         "reduceOptions": {
 340           "calcs": [
 341             "lastNotNull"
 342           ],
 343           "fields": "",
 344           "values": false
 345         },
 346         "text": {},
 347         "textMode": "auto"
 348       },
 349       "pluginVersion": "8.1.3",
 350       "targets": [
 351         {
 352           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 353           "format": "time_series",
 354           "instant": true,
 355           "interval": "",
 356           "intervalFactor": 1,
 357           "legendFormat": "",
 358           "refId": "A"
 359         }
 360       ],
 361       "title": "Preferred Replica Imbalance",
 362       "type": "stat"
 363     },
 364     {
 365       "datasource": "Prometheus",
 366       "fieldConfig": {
 367         "defaults": {
 368           "color": {
 369             "mode": "palette-classic"
 370           },
 371           "custom": {
 372             "axisLabel": "Bytes/s",
 373             "axisPlacement": "auto",
 374             "barAlignment": 0,
 375             "drawStyle": "line",
 376             "fillOpacity": 10,
 377             "gradientMode": "none",
 378             "hideFrom": {
 379               "legend": false,
 380               "tooltip": false,
 381               "viz": false
 382             },
 383             "lineInterpolation": "linear",
 384             "lineWidth": 2,
 385             "pointSize": 5,
 386             "scaleDistribution": {
 387               "type": "linear"
 388             },
 389             "showPoints": "never",
 390             "spanNulls": false,
 391             "stacking": {
 392               "group": "A",
 393               "mode": "none"
 394             },
 395             "thresholdsStyle": {
 396               "mode": "off"
 397             }
 398           },
 399           "links": [],
 400           "mappings": [],
 401           "thresholds": {
 402             "mode": "absolute",
 403             "steps": [
 404               {
 405                 "color": "green",
 406                 "value": null
 407               },
 408               {
 409                 "color": "red",
 410                 "value": 80
 411               }
 412             ]
 413           },
 414           "unit": "Bps"
 415         },
 416         "overrides": []
 417       },
 418       "gridPos": {
 419         "h": 8,
 420         "w": 8,
 421         "x": 16,
 422         "y": 1
 423       },
 424       "id": 84,
 425       "links": [],
 426       "options": {
 427         "legend": {
 428           "calcs": [
 429             "mean",
 430             "lastNotNull",
 431             "max"
 432           ],
 433           "displayMode": "list",
 434           "placement": "bottom"
 435         },
 436         "tooltip": {
 437           "mode": "single"
 438         }
 439       },
 440       "pluginVersion": "8.1.3",
 441       "targets": [
 442         {
 443           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 444           "format": "time_series",
 445           "interval": "",
 446           "intervalFactor": 2,
 447           "legendFormat": "Bytes in",
 448           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 449           "refId": "A",
 450           "step": 4
 451         },
 452         {
 453           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 454           "format": "time_series",
 455           "hide": false,
 456           "interval": "",
 457           "intervalFactor": 2,
 458           "legendFormat": "Bytes out",
 459           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 460           "refId": "B",
 461           "step": 4
 462         }
 463       ],
 464       "timeFrom": null,
 465       "timeShift": null,
 466       "title": "Broker network throughput",
 467       "type": "timeseries"
 468     },
 469     {
 470       "cacheTimeout": null,
 471       "datasource": "Prometheus",
 472       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 473       "fieldConfig": {
 474         "defaults": {
 475           "color": {
 476             "mode": "thresholds"
 477           },
 478           "mappings": [
 479             {
 480               "options": {
 481                 "match": "null",
 482                 "result": {
 483                   "text": "N/A"
 484                 }
 485               },
 486               "type": "special"
 487             }
 488           ],
 489           "thresholds": {
 490             "mode": "absolute",
 491             "steps": [
 492               {
 493                 "color": "#508642",
 494                 "value": null
 495               },
 496               {
 497                 "color": "rgba(237, 129, 40, 0.89)",
 498                 "value": 1
 499               },
 500               {
 501                 "color": "#bf1b00",
 502                 "value": 5
 503               }
 504             ]
 505           },
 506           "unit": "none"
 507         },
 508         "overrides": []
 509       },
 510       "gridPos": {
 511         "h": 4,
 512         "w": 4,
 513         "x": 0,
 514         "y": 5
 515       },
 516       "id": 20,
 517       "interval": null,
 518       "links": [],
 519       "maxDataPoints": 100,
 520       "options": {
 521         "colorMode": "value",
 522         "graphMode": "none",
 523         "justifyMode": "auto",
 524         "orientation": "horizontal",
 525         "reduceOptions": {
 526           "calcs": [
 527             "lastNotNull"
 528           ],
 529           "fields": "",
 530           "values": false
 531         },
 532         "text": {},
 533         "textMode": "auto"
 534       },
 535       "pluginVersion": "8.1.3",
 536       "targets": [
 537         {
 538           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 539           "format": "time_series",
 540           "hide": false,
 541           "instant": true,
 542           "interval": "",
 543           "intervalFactor": 2,
 544           "legendFormat": "",
 545           "refId": "A"
 546         }
 547       ],
 548       "title": "Under Replicated Partitions",
 549       "type": "stat"
 550     },
 551     {
 552       "cacheTimeout": null,
 553       "datasource": "Prometheus",
 554       "description": "Number of partitions under min insync replicas.",
 555       "fieldConfig": {
 556         "defaults": {
 557           "color": {
 558             "mode": "thresholds"
 559           },
 560           "mappings": [
 561             {
 562               "options": {
 563                 "match": "null",
 564                 "result": {
 565                   "text": "N/A"
 566                 }
 567               },
 568               "type": "special"
 569             }
 570           ],
 571           "thresholds": {
 572             "mode": "absolute",
 573             "steps": [
 574               {
 575                 "color": "#508642",
 576                 "value": null
 577               },
 578               {
 579                 "color": "rgba(237, 129, 40, 0.89)",
 580                 "value": 1
 581               },
 582               {
 583                 "color": "#bf1b00",
 584                 "value": 5
 585               }
 586             ]
 587           },
 588           "unit": "none"
 589         },
 590         "overrides": []
 591       },
 592       "gridPos": {
 593         "h": 4,
 594         "w": 4,
 595         "x": 4,
 596         "y": 5
 597       },
 598       "id": 32,
 599       "interval": null,
 600       "links": [],
 601       "maxDataPoints": 100,
 602       "options": {
 603         "colorMode": "value",
 604         "graphMode": "none",
 605         "justifyMode": "auto",
 606         "orientation": "horizontal",
 607         "reduceOptions": {
 608           "calcs": [
 609             "lastNotNull"
 610           ],
 611           "fields": "",
 612           "values": false
 613         },
 614         "text": {},
 615         "textMode": "auto"
 616       },
 617       "pluginVersion": "8.1.3",
 618       "targets": [
 619         {
 620           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 621           "format": "time_series",
 622           "hide": false,
 623           "instant": true,
 624           "interval": "",
 625           "intervalFactor": 2,
 626           "legendFormat": "",
 627           "refId": "A"
 628         }
 629       ],
 630       "title": "Under Min ISR Partitions",
 631       "type": "stat"
 632     },
 633     {
 634       "cacheTimeout": null,
 635       "datasource": "Prometheus",
 636       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 637       "fieldConfig": {
 638         "defaults": {
 639           "color": {
 640             "mode": "thresholds"
 641           },
 642           "mappings": [
 643             {
 644               "options": {
 645                 "match": "null",
 646                 "result": {
 647                   "text": "N/A"
 648                 }
 649               },
 650               "type": "special"
 651             }
 652           ],
 653           "thresholds": {
 654             "mode": "absolute",
 655             "steps": [
 656               {
 657                 "color": "#508642",
 658                 "value": null
 659               },
 660               {
 661                 "color": "#ef843c",
 662                 "value": 1
 663               },
 664               {
 665                 "color": "#bf1b00",
 666                 "value": 1
 667               }
 668             ]
 669           },
 670           "unit": "none"
 671         },
 672         "overrides": []
 673       },
 674       "gridPos": {
 675         "h": 4,
 676         "w": 4,
 677         "x": 8,
 678         "y": 5
 679       },
 680       "id": 22,
 681       "interval": null,
 682       "links": [],
 683       "maxDataPoints": 100,
 684       "options": {
 685         "colorMode": "value",
 686         "graphMode": "none",
 687         "justifyMode": "auto",
 688         "orientation": "horizontal",
 689         "reduceOptions": {
 690           "calcs": [
 691             "lastNotNull"
 692           ],
 693           "fields": "",
 694           "values": false
 695         },
 696         "text": {},
 697         "textMode": "auto"
 698       },
 699       "pluginVersion": "8.1.3",
 700       "targets": [
 701         {
 702           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 703           "format": "time_series",
 704           "instant": true,
 705           "interval": "",
 706           "intervalFactor": 1,
 707           "legendFormat": "",
 708           "refId": "A"
 709         }
 710       ],
 711       "title": "Offline Partitions Count",
 712       "type": "stat"
 713     },
 714     {
 715       "cacheTimeout": null,
 716       "datasource": "Prometheus",
 717       "description": "Unclean leader election rate",
 718       "fieldConfig": {
 719         "defaults": {
 720           "color": {
 721             "mode": "thresholds"
 722           },
 723           "mappings": [
 724             {
 725               "options": {
 726                 "match": "null",
 727                 "result": {
 728                   "text": "N/A"
 729                 }
 730               },
 731               "type": "special"
 732             }
 733           ],
 734           "thresholds": {
 735             "mode": "absolute",
 736             "steps": [
 737               {
 738                 "color": "#299c46",
 739                 "value": null
 740               },
 741               {
 742                 "color": "rgba(237, 129, 40, 0.89)",
 743                 "value": 2
 744               },
 745               {
 746                 "color": "#d44a3a"
 747               }
 748             ]
 749           },
 750           "unit": "none"
 751         },
 752         "overrides": []
 753       },
 754       "gridPos": {
 755         "h": 4,
 756         "w": 4,
 757         "x": 12,
 758         "y": 5
 759       },
 760       "id": 16,
 761       "interval": null,
 762       "links": [],
 763       "maxDataPoints": 100,
 764       "options": {
 765         "colorMode": "value",
 766         "graphMode": "none",
 767         "justifyMode": "auto",
 768         "orientation": "horizontal",
 769         "reduceOptions": {
 770           "calcs": [
 771             "lastNotNull"
 772           ],
 773           "fields": "",
 774           "values": false
 775         },
 776         "text": {},
 777         "textMode": "auto"
 778       },
 779       "pluginVersion": "8.1.3",
 780       "targets": [
 781         {
 782           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 783           "format": "time_series",
 784           "instant": true,
 785           "interval": "",
 786           "intervalFactor": 1,
 787           "legendFormat": "",
 788           "refId": "A"
 789         }
 790       ],
 791       "title": "Unclean Leader Election Rate",
 792       "type": "stat"
 793     },
 794     {
 795       "collapsed": false,
 796       "datasource": null,
 797       "gridPos": {
 798         "h": 1,
 799         "w": 24,
 800         "x": 0,
 801         "y": 9
 802       },
 803       "id": 31,
 804       "panels": [],
 805       "title": "Request rate",
 806       "type": "row"
 807     },
 808     {
 809       "cacheTimeout": null,
 810       "datasource": "Prometheus",
 811       "description": "Produce request rate.",
 812       "fieldConfig": {
 813         "defaults": {
 814           "mappings": [
 815             {
 816               "options": {
 817                 "match": "null",
 818                 "result": {
 819                   "text": "N/A"
 820                 }
 821               },
 822               "type": "special"
 823             }
 824           ],
 825           "thresholds": {
 826             "mode": "absolute",
 827             "steps": [
 828               {
 829                 "color": "green",
 830                 "value": null
 831               }
 832             ]
 833           },
 834           "unit": "none"
 835         },
 836         "overrides": []
 837       },
 838       "gridPos": {
 839         "h": 4,
 840         "w": 4,
 841         "x": 0,
 842         "y": 10
 843       },
 844       "id": 93,
 845       "interval": null,
 846       "links": [],
 847       "maxDataPoints": 100,
 848       "options": {
 849         "colorMode": "value",
 850         "graphMode": "area",
 851         "justifyMode": "auto",
 852         "orientation": "horizontal",
 853         "reduceOptions": {
 854           "calcs": [
 855             "lastNotNull"
 856           ],
 857           "fields": "",
 858           "values": false
 859         },
 860         "text": {},
 861         "textMode": "auto"
 862       },
 863       "pluginVersion": "8.1.3",
 864       "targets": [
 865         {
 866           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
 867           "interval": "",
 868           "legendFormat": "",
 869           "refId": "A"
 870         }
 871       ],
 872       "timeFrom": null,
 873       "timeShift": null,
 874       "title": "All Request Per Sec",
 875       "type": "stat"
 876     },
 877     {
 878       "cacheTimeout": null,
 879       "datasource": "Prometheus",
 880       "description": "Produce request rate.",
 881       "fieldConfig": {
 882         "defaults": {
 883           "mappings": [
 884             {
 885               "options": {
 886                 "match": "null",
 887                 "result": {
 888                   "text": "N/A"
 889                 }
 890               },
 891               "type": "special"
 892             }
 893           ],
 894           "thresholds": {
 895             "mode": "absolute",
 896             "steps": [
 897               {
 898                 "color": "green",
 899                 "value": null
 900               }
 901             ]
 902           },
 903           "unit": "none"
 904         },
 905         "overrides": []
 906       },
 907       "gridPos": {
 908         "h": 4,
 909         "w": 4,
 910         "x": 4,
 911         "y": 10
 912       },
 913       "id": 35,
 914       "interval": null,
 915       "links": [],
 916       "maxDataPoints": 100,
 917       "options": {
 918         "colorMode": "value",
 919         "graphMode": "area",
 920         "justifyMode": "auto",
 921         "orientation": "horizontal",
 922         "reduceOptions": {
 923           "calcs": [
 924             "lastNotNull"
 925           ],
 926           "fields": "",
 927           "values": false
 928         },
 929         "text": {},
 930         "textMode": "auto"
 931       },
 932       "pluginVersion": "8.1.3",
 933       "targets": [
 934         {
 935           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
 936           "interval": "",
 937           "legendFormat": "",
 938           "refId": "A"
 939         }
 940       ],
 941       "timeFrom": null,
 942       "timeShift": null,
 943       "title": "Produce Request Per Sec",
 944       "type": "stat"
 945     },
 946     {
 947       "cacheTimeout": null,
 948       "datasource": "Prometheus",
 949       "description": "Fetch request rate.",
 950       "fieldConfig": {
 951         "defaults": {
 952           "mappings": [
 953             {
 954               "options": {
 955                 "match": "null",
 956                 "result": {
 957                   "text": "N/A"
 958                 }
 959               },
 960               "type": "special"
 961             }
 962           ],
 963           "thresholds": {
 964             "mode": "absolute",
 965             "steps": [
 966               {
 967                 "color": "green",
 968                 "value": null
 969               }
 970             ]
 971           },
 972           "unit": "none"
 973         },
 974         "overrides": []
 975       },
 976       "gridPos": {
 977         "h": 4,
 978         "w": 4,
 979         "x": 8,
 980         "y": 10
 981       },
 982       "id": 37,
 983       "interval": null,
 984       "links": [],
 985       "maxDataPoints": 100,
 986       "options": {
 987         "colorMode": "value",
 988         "graphMode": "area",
 989         "justifyMode": "auto",
 990         "orientation": "horizontal",
 991         "reduceOptions": {
 992           "calcs": [
 993             "lastNotNull"
 994           ],
 995           "fields": "",
 996           "values": false
 997         },
 998         "text": {},
 999         "textMode": "auto"
1000       },
1001       "pluginVersion": "8.1.3",
1002       "targets": [
1003         {
1004           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1005           "interval": "",
1006           "legendFormat": "",
1007           "refId": "A"
1008         }
1009       ],
1010       "timeFrom": null,
1011       "timeShift": null,
1012       "title": "Consumer Fetch Request Per Sec",
1013       "type": "stat"
1014     },
1015     {
1016       "datasource": null,
1017       "fieldConfig": {
1018         "defaults": {
1019           "color": {
1020             "mode": "palette-classic"
1021           },
1022           "custom": {
1023             "axisLabel": "",
1024             "axisPlacement": "auto",
1025             "barAlignment": 0,
1026             "drawStyle": "line",
1027             "fillOpacity": 0,
1028             "gradientMode": "none",
1029             "hideFrom": {
1030               "legend": false,
1031               "tooltip": false,
1032               "viz": false
1033             },
1034             "lineInterpolation": "linear",
1035             "lineWidth": 1,
1036             "pointSize": 5,
1037             "scaleDistribution": {
1038               "type": "linear"
1039             },
1040             "showPoints": "auto",
1041             "spanNulls": false,
1042             "stacking": {
1043               "group": "A",
1044               "mode": "none"
1045             },
1046             "thresholdsStyle": {
1047               "mode": "off"
1048             }
1049           },
1050           "mappings": [],
1051           "thresholds": {
1052             "mode": "absolute",
1053             "steps": [
1054               {
1055                 "color": "green",
1056                 "value": null
1057               },
1058               {
1059                 "color": "red",
1060                 "value": 80
1061               }
1062             ]
1063           }
1064         },
1065         "overrides": []
1066       },
1067       "gridPos": {
1068         "h": 8,
1069         "w": 12,
1070         "x": 12,
1071         "y": 10
1072       },
1073       "id": 122,
1074       "options": {
1075         "legend": {
1076           "calcs": [
1077             "lastNotNull",
1078             "max",
1079             "mean"
1080           ],
1081           "displayMode": "table",
1082           "placement": "bottom"
1083         },
1084         "tooltip": {
1085           "mode": "single"
1086         }
1087       },
1088       "targets": [
1089         {
1090           "exemplar": true,
1091           "expr": "rate(kafka_network_requestmetrics_errorspersec{error!=\"NONE\"}[5m])",
1092           "interval": "",
1093           "legendFormat": "{{error}} @ {{hostname}}",
1094           "refId": "A"
1095         }
1096       ],
1097       "title": "Errors",
1098       "type": "timeseries"
1099     },
1100     {
1101       "cacheTimeout": null,
1102       "datasource": "Prometheus",
1103       "description": "Fetch request rate.",
1104       "fieldConfig": {
1105         "defaults": {
1106           "mappings": [
1107             {
1108               "options": {
1109                 "match": "null",
1110                 "result": {
1111                   "text": "N/A"
1112                 }
1113               },
1114               "type": "special"
1115             }
1116           ],
1117           "thresholds": {
1118             "mode": "absolute",
1119             "steps": [
1120               {
1121                 "color": "green",
1122                 "value": null
1123               }
1124             ]
1125           },
1126           "unit": "none"
1127         },
1128         "overrides": []
1129       },
1130       "gridPos": {
1131         "h": 4,
1132         "w": 4,
1133         "x": 0,
1134         "y": 14
1135       },
1136       "id": 94,
1137       "interval": null,
1138       "links": [],
1139       "maxDataPoints": 100,
1140       "options": {
1141         "colorMode": "value",
1142         "graphMode": "area",
1143         "justifyMode": "auto",
1144         "orientation": "horizontal",
1145         "reduceOptions": {
1146           "calcs": [
1147             "lastNotNull"
1148           ],
1149           "fields": "",
1150           "values": false
1151         },
1152         "text": {},
1153         "textMode": "auto"
1154       },
1155       "pluginVersion": "8.1.3",
1156       "targets": [
1157         {
1158           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1159           "interval": "",
1160           "legendFormat": "",
1161           "refId": "A"
1162         }
1163       ],
1164       "timeFrom": null,
1165       "timeShift": null,
1166       "title": "Broker Fetch Request Per Sec",
1167       "type": "stat"
1168     },
1169     {
1170       "cacheTimeout": null,
1171       "datasource": "Prometheus",
1172       "description": "Offset Commit request rate.",
1173       "fieldConfig": {
1174         "defaults": {
1175           "mappings": [
1176             {
1177               "options": {
1178                 "match": "null",
1179                 "result": {
1180                   "text": "N/A"
1181                 }
1182               },
1183               "type": "special"
1184             }
1185           ],
1186           "thresholds": {
1187             "mode": "absolute",
1188             "steps": [
1189               {
1190                 "color": "green",
1191                 "value": null
1192               }
1193             ]
1194           },
1195           "unit": "none"
1196         },
1197         "overrides": []
1198       },
1199       "gridPos": {
1200         "h": 4,
1201         "w": 4,
1202         "x": 4,
1203         "y": 14
1204       },
1205       "id": 38,
1206       "interval": null,
1207       "links": [],
1208       "maxDataPoints": 100,
1209       "options": {
1210         "colorMode": "value",
1211         "graphMode": "area",
1212         "justifyMode": "auto",
1213         "orientation": "horizontal",
1214         "reduceOptions": {
1215           "calcs": [
1216             "lastNotNull"
1217           ],
1218           "fields": "",
1219           "values": false
1220         },
1221         "text": {},
1222         "textMode": "auto"
1223       },
1224       "pluginVersion": "8.1.3",
1225       "targets": [
1226         {
1227           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1228           "interval": "",
1229           "legendFormat": "",
1230           "refId": "A"
1231         }
1232       ],
1233       "timeFrom": null,
1234       "timeShift": null,
1235       "title": "Offset Commit Request Per Sec",
1236       "type": "stat"
1237     },
1238     {
1239       "cacheTimeout": null,
1240       "datasource": "Prometheus",
1241       "description": "Metadata request rate.",
1242       "fieldConfig": {
1243         "defaults": {
1244           "mappings": [
1245             {
1246               "options": {
1247                 "match": "null",
1248                 "result": {
1249                   "text": "N/A"
1250                 }
1251               },
1252               "type": "special"
1253             }
1254           ],
1255           "thresholds": {
1256             "mode": "absolute",
1257             "steps": [
1258               {
1259                 "color": "green",
1260                 "value": null
1261               }
1262             ]
1263           },
1264           "unit": "none"
1265         },
1266         "overrides": []
1267       },
1268       "gridPos": {
1269         "h": 4,
1270         "w": 4,
1271         "x": 8,
1272         "y": 14
1273       },
1274       "id": 36,
1275       "interval": null,
1276       "links": [],
1277       "maxDataPoints": 100,
1278       "options": {
1279         "colorMode": "value",
1280         "graphMode": "area",
1281         "justifyMode": "auto",
1282         "orientation": "horizontal",
1283         "reduceOptions": {
1284           "calcs": [
1285             "lastNotNull"
1286           ],
1287           "fields": "",
1288           "values": false
1289         },
1290         "text": {},
1291         "textMode": "auto"
1292       },
1293       "pluginVersion": "8.1.3",
1294       "targets": [
1295         {
1296           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1297           "interval": "",
1298           "legendFormat": "",
1299           "refId": "A"
1300         }
1301       ],
1302       "timeFrom": null,
1303       "timeShift": null,
1304       "title": "Metadata Request Per Sec",
1305       "type": "stat"
1306     },
1307     {
1308       "collapsed": false,
1309       "datasource": null,
1310       "gridPos": {
1311         "h": 1,
1312         "w": 24,
1313         "x": 0,
1314         "y": 18
1315       },
1316       "id": 40,
1317       "panels": [],
1318       "title": "System",
1319       "type": "row"
1320     },
1321     {
1322       "datasource": "Prometheus",
1323       "fieldConfig": {
1324         "defaults": {
1325           "color": {
1326             "mode": "palette-classic"
1327           },
1328           "custom": {
1329             "axisLabel": "Cores",
1330             "axisPlacement": "auto",
1331             "barAlignment": 0,
1332             "drawStyle": "line",
1333             "fillOpacity": 10,
1334             "gradientMode": "none",
1335             "hideFrom": {
1336               "legend": false,
1337               "tooltip": false,
1338               "viz": false
1339             },
1340             "lineInterpolation": "linear",
1341             "lineWidth": 2,
1342             "pointSize": 5,
1343             "scaleDistribution": {
1344               "type": "linear"
1345             },
1346             "showPoints": "never",
1347             "spanNulls": false,
1348             "stacking": {
1349               "group": "A",
1350               "mode": "none"
1351             },
1352             "thresholdsStyle": {
1353               "mode": "off"
1354             }
1355           },
1356           "links": [],
1357           "mappings": [],
1358           "thresholds": {
1359             "mode": "absolute",
1360             "steps": [
1361               {
1362                 "color": "green",
1363                 "value": null
1364               },
1365               {
1366                 "color": "red",
1367                 "value": 80
1368               }
1369             ]
1370           },
1371           "unit": "percent"
1372         },
1373         "overrides": [
1374           {
1375             "matcher": {
1376               "id": "byName",
1377               "options": "localhost:7071"
1378             },
1379             "properties": [
1380               {
1381                 "id": "color",
1382                 "value": {
1383                   "fixedColor": "#629E51",
1384                   "mode": "fixed"
1385                 }
1386               }
1387             ]
1388           }
1389         ]
1390       },
1391       "gridPos": {
1392         "h": 8,
1393         "w": 8,
1394         "x": 0,
1395         "y": 19
1396       },
1397       "id": 27,
1398       "links": [],
1399       "options": {
1400         "legend": {
1401           "calcs": [
1402             "mean",
1403             "lastNotNull",
1404             "max"
1405           ],
1406           "displayMode": "table",
1407           "placement": "bottom"
1408         },
1409         "tooltip": {
1410           "mode": "single"
1411         }
1412       },
1413       "pluginVersion": "8.1.3",
1414       "targets": [
1415         {
1416           "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1417           "format": "time_series",
1418           "interval": "",
1419           "intervalFactor": 2,
1420           "legendFormat": "{{instance}}",
1421           "metric": "process_cpu_secondspersec",
1422           "refId": "A",
1423           "step": 4
1424         }
1425       ],
1426       "timeFrom": null,
1427       "timeShift": null,
1428       "title": "CPU Usage",
1429       "type": "timeseries"
1430     },
1431     {
1432       "datasource": "Prometheus",
1433       "fieldConfig": {
1434         "defaults": {
1435           "color": {
1436             "mode": "palette-classic"
1437           },
1438           "custom": {
1439             "axisLabel": "Memory",
1440             "axisPlacement": "auto",
1441             "barAlignment": 0,
1442             "drawStyle": "line",
1443             "fillOpacity": 10,
1444             "gradientMode": "none",
1445             "hideFrom": {
1446               "legend": false,
1447               "tooltip": false,
1448               "viz": false
1449             },
1450             "lineInterpolation": "linear",
1451             "lineWidth": 2,
1452             "pointSize": 5,
1453             "scaleDistribution": {
1454               "type": "linear"
1455             },
1456             "showPoints": "never",
1457             "spanNulls": false,
1458             "stacking": {
1459               "group": "A",
1460               "mode": "none"
1461             },
1462             "thresholdsStyle": {
1463               "mode": "off"
1464             }
1465           },
1466           "links": [],
1467           "mappings": [],
1468           "thresholds": {
1469             "mode": "absolute",
1470             "steps": [
1471               {
1472                 "color": "green",
1473                 "value": null
1474               },
1475               {
1476                 "color": "red",
1477                 "value": 80
1478               }
1479             ]
1480           },
1481           "unit": "bytes"
1482         },
1483         "overrides": [
1484           {
1485             "matcher": {
1486               "id": "byName",
1487               "options": "localhost:7071"
1488             },
1489             "properties": [
1490               {
1491                 "id": "color",
1492                 "value": {
1493                   "fixedColor": "#BA43A9",
1494                   "mode": "fixed"
1495                 }
1496               }
1497             ]
1498           }
1499         ]
1500       },
1501       "gridPos": {
1502         "h": 8,
1503         "w": 8,
1504         "x": 8,
1505         "y": 19
1506       },
1507       "id": 2,
1508       "links": [],
1509       "options": {
1510         "legend": {
1511           "calcs": [
1512             "mean",
1513             "lastNotNull",
1514             "max"
1515           ],
1516           "displayMode": "table",
1517           "placement": "bottom"
1518         },
1519         "tooltip": {
1520           "mode": "single"
1521         }
1522       },
1523       "pluginVersion": "8.1.3",
1524       "targets": [
1525         {
1526           "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1527           "interval": "",
1528           "intervalFactor": 2,
1529           "legendFormat": "{{instance}}",
1530           "metric": "jvm_memory_bytes_used",
1531           "refId": "A",
1532           "step": 4
1533         },
1534         {
1535           "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1536           "interval": "",
1537           "legendFormat": "{{instance}}",
1538           "refId": "B"
1539         }
1540       ],
1541       "timeFrom": null,
1542       "timeShift": null,
1543       "title": "JVM Memory Used",
1544       "type": "timeseries"
1545     },
1546     {
1547       "datasource": "Prometheus",
1548       "fieldConfig": {
1549         "defaults": {
1550           "color": {
1551             "mode": "palette-classic"
1552           },
1553           "custom": {
1554             "axisLabel": "% time in GC",
1555             "axisPlacement": "auto",
1556             "barAlignment": 0,
1557             "drawStyle": "line",
1558             "fillOpacity": 10,
1559             "gradientMode": "none",
1560             "hideFrom": {
1561               "legend": false,
1562               "tooltip": false,
1563               "viz": false
1564             },
1565             "lineInterpolation": "linear",
1566             "lineWidth": 2,
1567             "pointSize": 5,
1568             "scaleDistribution": {
1569               "type": "linear"
1570             },
1571             "showPoints": "never",
1572             "spanNulls": false,
1573             "stacking": {
1574               "group": "A",
1575               "mode": "none"
1576             },
1577             "thresholdsStyle": {
1578               "mode": "off"
1579             }
1580           },
1581           "links": [],
1582           "mappings": [],
1583           "thresholds": {
1584             "mode": "absolute",
1585             "steps": [
1586               {
1587                 "color": "green",
1588                 "value": null
1589               },
1590               {
1591                 "color": "red",
1592                 "value": 80
1593               }
1594             ]
1595           },
1596           "unit": "percentunit"
1597         },
1598         "overrides": [
1599           {
1600             "matcher": {
1601               "id": "byName",
1602               "options": "localhost:7071"
1603             },
1604             "properties": [
1605               {
1606                 "id": "color",
1607                 "value": {
1608                   "fixedColor": "#890F02",
1609                   "mode": "fixed"
1610                 }
1611               }
1612             ]
1613           }
1614         ]
1615       },
1616       "gridPos": {
1617         "h": 8,
1618         "w": 8,
1619         "x": 16,
1620         "y": 19
1621       },
1622       "id": 3,
1623       "links": [],
1624       "options": {
1625         "legend": {
1626           "calcs": [
1627             "mean",
1628             "lastNotNull",
1629             "max"
1630           ],
1631           "displayMode": "table",
1632           "placement": "bottom"
1633         },
1634         "tooltip": {
1635           "mode": "single"
1636         }
1637       },
1638       "pluginVersion": "8.1.3",
1639       "targets": [
1640         {
1641           "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1642           "interval": "",
1643           "intervalFactor": 2,
1644           "legendFormat": "{{instance}}",
1645           "metric": "jvm_gc_collection_seconds_sum",
1646           "refId": "A",
1647           "step": 4
1648         }
1649       ],
1650       "timeFrom": null,
1651       "timeShift": null,
1652       "title": "Time spent in GC",
1653       "type": "timeseries"
1654     },
1655     {
1656       "collapsed": true,
1657       "datasource": null,
1658       "gridPos": {
1659         "h": 1,
1660         "w": 24,
1661         "x": 0,
1662         "y": 27
1663       },
1664       "id": 29,
1665       "panels": [
1666         {
1667           "datasource": "Prometheus",
1668           "fieldConfig": {
1669             "defaults": {
1670               "color": {
1671                 "mode": "palette-classic"
1672               },
1673               "custom": {
1674                 "axisLabel": "Messages/s",
1675                 "axisPlacement": "auto",
1676                 "barAlignment": 0,
1677                 "drawStyle": "line",
1678                 "fillOpacity": 10,
1679                 "gradientMode": "none",
1680                 "hideFrom": {
1681                   "legend": false,
1682                   "tooltip": false,
1683                   "viz": false
1684                 },
1685                 "lineInterpolation": "linear",
1686                 "lineWidth": 2,
1687                 "pointSize": 5,
1688                 "scaleDistribution": {
1689                   "type": "linear"
1690                 },
1691                 "showPoints": "never",
1692                 "spanNulls": false,
1693                 "stacking": {
1694                   "group": "A",
1695                   "mode": "none"
1696                 },
1697                 "thresholdsStyle": {
1698                   "mode": "off"
1699                 }
1700               },
1701               "links": [],
1702               "mappings": [],
1703               "min": 0,
1704               "thresholds": {
1705                 "mode": "absolute",
1706                 "steps": [
1707                   {
1708                     "color": "green",
1709                     "value": null
1710                   }
1711                 ]
1712               },
1713               "unit": "cps"
1714             },
1715             "overrides": []
1716           },
1717           "gridPos": {
1718             "h": 8,
1719             "w": 8,
1720             "x": 0,
1721             "y": 24
1722           },
1723           "id": 4,
1724           "links": [],
1725           "options": {
1726             "legend": {
1727               "calcs": [
1728                 "mean",
1729                 "lastNotNull",
1730                 "max"
1731               ],
1732               "displayMode": "table",
1733               "placement": "bottom"
1734             },
1735             "tooltip": {
1736               "mode": "single"
1737             }
1738           },
1739           "pluginVersion": "8.1.3",
1740           "targets": [
1741             {
1742               "exemplar": true,
1743               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1744               "interval": "",
1745               "intervalFactor": 2,
1746               "legendFormat": "bytes/sec",
1747               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
1748               "refId": "A",
1749               "step": 4
1750             }
1751           ],
1752           "timeFrom": null,
1753           "timeShift": null,
1754           "title": "Messages In",
1755           "type": "timeseries"
1756         },
1757         {
1758           "datasource": "Prometheus",
1759           "fieldConfig": {
1760             "defaults": {
1761               "color": {
1762                 "mode": "palette-classic"
1763               },
1764               "custom": {
1765                 "axisLabel": "Bytes/s",
1766                 "axisPlacement": "auto",
1767                 "barAlignment": 0,
1768                 "drawStyle": "line",
1769                 "fillOpacity": 10,
1770                 "gradientMode": "none",
1771                 "hideFrom": {
1772                   "legend": false,
1773                   "tooltip": false,
1774                   "viz": false
1775                 },
1776                 "lineInterpolation": "linear",
1777                 "lineWidth": 2,
1778                 "pointSize": 5,
1779                 "scaleDistribution": {
1780                   "type": "linear"
1781                 },
1782                 "showPoints": "never",
1783                 "spanNulls": false,
1784                 "stacking": {
1785                   "group": "A",
1786                   "mode": "none"
1787                 },
1788                 "thresholdsStyle": {
1789                   "mode": "off"
1790                 }
1791               },
1792               "links": [],
1793               "mappings": [],
1794               "min": 0,
1795               "thresholds": {
1796                 "mode": "absolute",
1797                 "steps": [
1798                   {
1799                     "color": "green",
1800                     "value": null
1801                   },
1802                   {
1803                     "color": "red",
1804                     "value": 80
1805                   }
1806                 ]
1807               },
1808               "unit": "Bps"
1809             },
1810             "overrides": []
1811           },
1812           "gridPos": {
1813             "h": 8,
1814             "w": 8,
1815             "x": 8,
1816             "y": 24
1817           },
1818           "id": 5,
1819           "links": [],
1820           "options": {
1821             "legend": {
1822               "calcs": [
1823                 "mean",
1824                 "lastNotNull",
1825                 "max"
1826               ],
1827               "displayMode": "table",
1828               "placement": "bottom"
1829             },
1830             "tooltip": {
1831               "mode": "single"
1832             }
1833           },
1834           "pluginVersion": "8.1.3",
1835           "targets": [
1836             {
1837               "exemplar": true,
1838               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1839               "format": "time_series",
1840               "interval": "",
1841               "intervalFactor": 2,
1842               "legendFormat": "bytes/sec",
1843               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1844               "refId": "A",
1845               "step": 4
1846             }
1847           ],
1848           "timeFrom": null,
1849           "timeShift": null,
1850           "title": "Bytes In",
1851           "type": "timeseries"
1852         },
1853         {
1854           "datasource": "Prometheus",
1855           "fieldConfig": {
1856             "defaults": {
1857               "color": {
1858                 "mode": "palette-classic"
1859               },
1860               "custom": {
1861                 "axisLabel": "Bytes/s",
1862                 "axisPlacement": "auto",
1863                 "barAlignment": 0,
1864                 "drawStyle": "line",
1865                 "fillOpacity": 10,
1866                 "gradientMode": "none",
1867                 "hideFrom": {
1868                   "legend": false,
1869                   "tooltip": false,
1870                   "viz": false
1871                 },
1872                 "lineInterpolation": "linear",
1873                 "lineWidth": 2,
1874                 "pointSize": 5,
1875                 "scaleDistribution": {
1876                   "type": "linear"
1877                 },
1878                 "showPoints": "never",
1879                 "spanNulls": false,
1880                 "stacking": {
1881                   "group": "A",
1882                   "mode": "none"
1883                 },
1884                 "thresholdsStyle": {
1885                   "mode": "off"
1886                 }
1887               },
1888               "links": [],
1889               "mappings": [],
1890               "min": 0,
1891               "thresholds": {
1892                 "mode": "absolute",
1893                 "steps": [
1894                   {
1895                     "color": "green",
1896                     "value": null
1897                   },
1898                   {
1899                     "color": "red",
1900                     "value": 80
1901                   }
1902                 ]
1903               },
1904               "unit": "Bps"
1905             },
1906             "overrides": []
1907           },
1908           "gridPos": {
1909             "h": 8,
1910             "w": 8,
1911             "x": 16,
1912             "y": 24
1913           },
1914           "id": 6,
1915           "links": [],
1916           "options": {
1917             "legend": {
1918               "calcs": [
1919                 "mean",
1920                 "lastNotNull",
1921                 "max"
1922               ],
1923               "displayMode": "table",
1924               "placement": "bottom"
1925             },
1926             "tooltip": {
1927               "mode": "single"
1928             }
1929           },
1930           "pluginVersion": "8.1.3",
1931           "targets": [
1932             {
1933               "exemplar": true,
1934               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1935               "interval": "",
1936               "intervalFactor": 2,
1937               "legendFormat": "bytes/sec",
1938               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1939               "refId": "A",
1940               "step": 4
1941             }
1942           ],
1943           "timeFrom": null,
1944           "timeShift": null,
1945           "title": "Bytes Out",
1946           "type": "timeseries"
1947         },
1948         {
1949           "datasource": "Prometheus",
1950           "fieldConfig": {
1951             "defaults": {
1952               "color": {
1953                 "mode": "palette-classic"
1954               },
1955               "custom": {
1956                 "axisLabel": "Messages/s",
1957                 "axisPlacement": "auto",
1958                 "barAlignment": 0,
1959                 "drawStyle": "line",
1960                 "fillOpacity": 10,
1961                 "gradientMode": "none",
1962                 "hideFrom": {
1963                   "legend": false,
1964                   "tooltip": false,
1965                   "viz": false
1966                 },
1967                 "lineInterpolation": "linear",
1968                 "lineWidth": 2,
1969                 "pointSize": 5,
1970                 "scaleDistribution": {
1971                   "type": "linear"
1972                 },
1973                 "showPoints": "never",
1974                 "spanNulls": false,
1975                 "stacking": {
1976                   "group": "A",
1977                   "mode": "none"
1978                 },
1979                 "thresholdsStyle": {
1980                   "mode": "off"
1981                 }
1982               },
1983               "links": [],
1984               "mappings": [],
1985               "thresholds": {
1986                 "mode": "absolute",
1987                 "steps": [
1988                   {
1989                     "color": "green",
1990                     "value": null
1991                   },
1992                   {
1993                     "color": "red",
1994                     "value": 80
1995                   }
1996                 ]
1997               },
1998               "unit": "cps"
1999             },
2000             "overrides": []
2001           },
2002           "gridPos": {
2003             "h": 8,
2004             "w": 8,
2005             "x": 0,
2006             "y": 32
2007           },
2008           "id": 10,
2009           "links": [],
2010           "options": {
2011             "legend": {
2012               "calcs": [
2013                 "mean",
2014                 "lastNotNull",
2015                 "max"
2016               ],
2017               "displayMode": "table",
2018               "placement": "bottom"
2019             },
2020             "tooltip": {
2021               "mode": "single"
2022             }
2023           },
2024           "pluginVersion": "8.1.3",
2025           "targets": [
2026             {
2027               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2028               "format": "time_series",
2029               "interval": "",
2030               "intervalFactor": 2,
2031               "legendFormat": "{{instance}}",
2032               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2033               "refId": "A",
2034               "step": 4
2035             }
2036           ],
2037           "timeFrom": null,
2038           "timeShift": null,
2039           "title": "Messages In Per Broker",
2040           "type": "timeseries"
2041         },
2042         {
2043           "datasource": "Prometheus",
2044           "fieldConfig": {
2045             "defaults": {
2046               "color": {
2047                 "mode": "palette-classic"
2048               },
2049               "custom": {
2050                 "axisLabel": "Bytes/s",
2051                 "axisPlacement": "auto",
2052                 "barAlignment": 0,
2053                 "drawStyle": "line",
2054                 "fillOpacity": 10,
2055                 "gradientMode": "none",
2056                 "hideFrom": {
2057                   "legend": false,
2058                   "tooltip": false,
2059                   "viz": false
2060                 },
2061                 "lineInterpolation": "linear",
2062                 "lineWidth": 2,
2063                 "pointSize": 5,
2064                 "scaleDistribution": {
2065                   "type": "linear"
2066                 },
2067                 "showPoints": "never",
2068                 "spanNulls": false,
2069                 "stacking": {
2070                   "group": "A",
2071                   "mode": "none"
2072                 },
2073                 "thresholdsStyle": {
2074                   "mode": "off"
2075                 }
2076               },
2077               "links": [],
2078               "mappings": [],
2079               "thresholds": {
2080                 "mode": "absolute",
2081                 "steps": [
2082                   {
2083                     "color": "green",
2084                     "value": null
2085                   },
2086                   {
2087                     "color": "red",
2088                     "value": 80
2089                   }
2090                 ]
2091               },
2092               "unit": "Bps"
2093             },
2094             "overrides": []
2095           },
2096           "gridPos": {
2097             "h": 8,
2098             "w": 8,
2099             "x": 8,
2100             "y": 32
2101           },
2102           "id": 7,
2103           "links": [],
2104           "options": {
2105             "legend": {
2106               "calcs": [
2107                 "mean",
2108                 "lastNotNull",
2109                 "max"
2110               ],
2111               "displayMode": "table",
2112               "placement": "bottom"
2113             },
2114             "tooltip": {
2115               "mode": "single"
2116             }
2117           },
2118           "pluginVersion": "8.1.3",
2119           "targets": [
2120             {
2121               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2122               "format": "time_series",
2123               "interval": "",
2124               "intervalFactor": 2,
2125               "legendFormat": "{{instance}}",
2126               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2127               "refId": "A",
2128               "step": 4
2129             }
2130           ],
2131           "timeFrom": null,
2132           "timeShift": null,
2133           "title": "Bytes In Per Broker",
2134           "type": "timeseries"
2135         },
2136         {
2137           "datasource": "Prometheus",
2138           "fieldConfig": {
2139             "defaults": {
2140               "color": {
2141                 "mode": "palette-classic"
2142               },
2143               "custom": {
2144                 "axisLabel": "",
2145                 "axisPlacement": "auto",
2146                 "barAlignment": 0,
2147                 "drawStyle": "line",
2148                 "fillOpacity": 10,
2149                 "gradientMode": "none",
2150                 "hideFrom": {
2151                   "legend": false,
2152                   "tooltip": false,
2153                   "viz": false
2154                 },
2155                 "lineInterpolation": "linear",
2156                 "lineWidth": 1,
2157                 "pointSize": 5,
2158                 "scaleDistribution": {
2159                   "type": "linear"
2160                 },
2161                 "showPoints": "never",
2162                 "spanNulls": true,
2163                 "stacking": {
2164                   "group": "A",
2165                   "mode": "none"
2166                 },
2167                 "thresholdsStyle": {
2168                   "mode": "off"
2169                 }
2170               },
2171               "links": [],
2172               "mappings": [],
2173               "min": 0,
2174               "thresholds": {
2175                 "mode": "absolute",
2176                 "steps": [
2177                   {
2178                     "color": "green",
2179                     "value": null
2180                   },
2181                   {
2182                     "color": "red",
2183                     "value": 80
2184                   }
2185                 ]
2186               },
2187               "unit": "binBps"
2188             },
2189             "overrides": []
2190           },
2191           "gridPos": {
2192             "h": 8,
2193             "w": 8,
2194             "x": 16,
2195             "y": 32
2196           },
2197           "id": 9,
2198           "links": [],
2199           "options": {
2200             "legend": {
2201               "calcs": [
2202                 "mean",
2203                 "lastNotNull",
2204                 "max"
2205               ],
2206               "displayMode": "table",
2207               "placement": "bottom"
2208             },
2209             "tooltip": {
2210               "mode": "single"
2211             }
2212           },
2213           "pluginVersion": "8.1.3",
2214           "targets": [
2215             {
2216               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2217               "format": "time_series",
2218               "interval": "",
2219               "intervalFactor": 1,
2220               "legendFormat": "{{instance}}",
2221               "refId": "A"
2222             }
2223           ],
2224           "timeFrom": null,
2225           "timeShift": null,
2226           "title": "Bytes Out Per Broker",
2227           "type": "timeseries"
2228         }
2229       ],
2230       "title": "Throughput In/Out",
2231       "type": "row"
2232     },
2233     {
2234       "collapsed": true,
2235       "datasource": null,
2236       "gridPos": {
2237         "h": 1,
2238         "w": 24,
2239         "x": 0,
2240         "y": 28
2241       },
2242       "id": 44,
2243       "panels": [
2244         {
2245           "datasource": "Prometheus",
2246           "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2247           "fieldConfig": {
2248             "defaults": {
2249               "color": {
2250                 "mode": "palette-classic"
2251               },
2252               "custom": {
2253                 "axisLabel": "",
2254                 "axisPlacement": "auto",
2255                 "barAlignment": 0,
2256                 "drawStyle": "line",
2257                 "fillOpacity": 10,
2258                 "gradientMode": "none",
2259                 "hideFrom": {
2260                   "legend": false,
2261                   "tooltip": false,
2262                   "viz": false
2263                 },
2264                 "lineInterpolation": "linear",
2265                 "lineWidth": 1,
2266                 "pointSize": 5,
2267                 "scaleDistribution": {
2268                   "type": "linear"
2269                 },
2270                 "showPoints": "never",
2271                 "spanNulls": true,
2272                 "stacking": {
2273                   "group": "A",
2274                   "mode": "none"
2275                 },
2276                 "thresholdsStyle": {
2277                   "mode": "off"
2278                 }
2279               },
2280               "links": [],
2281               "mappings": [],
2282               "thresholds": {
2283                 "mode": "absolute",
2284                 "steps": [
2285                   {
2286                     "color": "green",
2287                     "value": null
2288                   },
2289                   {
2290                     "color": "red",
2291                     "value": 80
2292                   }
2293                 ]
2294               },
2295               "unit": "percentunit"
2296             },
2297             "overrides": []
2298           },
2299           "gridPos": {
2300             "h": 9,
2301             "w": 12,
2302             "x": 0,
2303             "y": 25
2304           },
2305           "id": 24,
2306           "options": {
2307             "legend": {
2308               "calcs": [
2309                 "mean",
2310                 "lastNotNull",
2311                 "max"
2312               ],
2313               "displayMode": "table",
2314               "placement": "bottom"
2315             },
2316             "tooltip": {
2317               "mode": "single"
2318             }
2319           },
2320           "pluginVersion": "8.1.3",
2321           "targets": [
2322             {
2323               "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2324               "interval": "",
2325               "legendFormat": "{{instance}}",
2326               "refId": "A"
2327             }
2328           ],
2329           "timeFrom": null,
2330           "timeShift": null,
2331           "title": "Network Processor Avg Usage Percent",
2332           "type": "timeseries"
2333         },
2334         {
2335           "datasource": "Prometheus",
2336           "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2337           "fieldConfig": {
2338             "defaults": {
2339               "color": {
2340                 "mode": "palette-classic"
2341               },
2342               "custom": {
2343                 "axisLabel": "",
2344                 "axisPlacement": "auto",
2345                 "barAlignment": 0,
2346                 "drawStyle": "line",
2347                 "fillOpacity": 10,
2348                 "gradientMode": "none",
2349                 "hideFrom": {
2350                   "legend": false,
2351                   "tooltip": false,
2352                   "viz": false
2353                 },
2354                 "lineInterpolation": "linear",
2355                 "lineWidth": 1,
2356                 "pointSize": 5,
2357                 "scaleDistribution": {
2358                   "type": "linear"
2359                 },
2360                 "showPoints": "never",
2361                 "spanNulls": true,
2362                 "stacking": {
2363                   "group": "A",
2364                   "mode": "none"
2365                 },
2366                 "thresholdsStyle": {
2367                   "mode": "off"
2368                 }
2369               },
2370               "links": [],
2371               "mappings": [],
2372               "min": 0,
2373               "thresholds": {
2374                 "mode": "absolute",
2375                 "steps": [
2376                   {
2377                     "color": "green",
2378                     "value": null
2379                   },
2380                   {
2381                     "color": "red",
2382                     "value": 80
2383                   }
2384                 ]
2385               },
2386               "unit": "percentunit"
2387             },
2388             "overrides": []
2389           },
2390           "gridPos": {
2391             "h": 9,
2392             "w": 12,
2393             "x": 12,
2394             "y": 25
2395           },
2396           "id": 25,
2397           "options": {
2398             "legend": {
2399               "calcs": [
2400                 "mean",
2401                 "lastNotNull",
2402                 "max"
2403               ],
2404               "displayMode": "table",
2405               "placement": "bottom"
2406             },
2407             "tooltip": {
2408               "mode": "single"
2409             }
2410           },
2411           "pluginVersion": "8.1.3",
2412           "targets": [
2413             {
2414               "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2415               "interval": "",
2416               "legendFormat": "{{instance}}",
2417               "refId": "A"
2418             }
2419           ],
2420           "timeFrom": null,
2421           "timeShift": null,
2422           "title": "Request Handler Avg Percent",
2423           "type": "timeseries"
2424         }
2425       ],
2426       "title": "Thread utilization",
2427       "type": "row"
2428     },
2429     {
2430       "collapsed": true,
2431       "datasource": null,
2432       "gridPos": {
2433         "h": 1,
2434         "w": 24,
2435         "x": 0,
2436         "y": 29
2437       },
2438       "id": 86,
2439       "panels": [
2440         {
2441           "datasource": "Prometheus",
2442           "description": "Latency in millseconds for ZooKeeper requests from broker.\n",
2443           "fieldConfig": {
2444             "defaults": {
2445               "color": {
2446                 "mode": "palette-classic"
2447               },
2448               "custom": {
2449                 "axisLabel": "",
2450                 "axisPlacement": "auto",
2451                 "barAlignment": 0,
2452                 "drawStyle": "line",
2453                 "fillOpacity": 10,
2454                 "gradientMode": "none",
2455                 "hideFrom": {
2456                   "legend": false,
2457                   "tooltip": false,
2458                   "viz": false
2459                 },
2460                 "lineInterpolation": "linear",
2461                 "lineWidth": 1,
2462                 "pointSize": 5,
2463                 "scaleDistribution": {
2464                   "type": "linear"
2465                 },
2466                 "showPoints": "never",
2467                 "spanNulls": true,
2468                 "stacking": {
2469                   "group": "A",
2470                   "mode": "none"
2471                 },
2472                 "thresholdsStyle": {
2473                   "mode": "off"
2474                 }
2475               },
2476               "links": [],
2477               "mappings": [],
2478               "thresholds": {
2479                 "mode": "absolute",
2480                 "steps": [
2481                   {
2482                     "color": "green",
2483                     "value": null
2484                   },
2485                   {
2486                     "color": "red",
2487                     "value": 80
2488                   }
2489                 ]
2490               },
2491               "unit": "ms"
2492             },
2493             "overrides": []
2494           },
2495           "gridPos": {
2496             "h": 9,
2497             "w": 12,
2498             "x": 0,
2499             "y": 26
2500           },
2501           "id": 88,
2502           "options": {
2503             "legend": {
2504               "calcs": [
2505                 "mean",
2506                 "lastNotNull",
2507                 "max"
2508               ],
2509               "displayMode": "table",
2510               "placement": "bottom"
2511             },
2512             "tooltip": {
2513               "mode": "single"
2514             }
2515           },
2516           "pluginVersion": "8.1.3",
2517           "targets": [
2518             {
2519               "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}",
2520               "hide": false,
2521               "legendFormat": "{{instance}} - {{quantile}}",
2522               "refId": "A"
2523             }
2524           ],
2525           "timeFrom": null,
2526           "timeShift": null,
2527           "title": "Zookeeper Request Latency",
2528           "type": "timeseries"
2529         },
2530         {
2531           "datasource": "Prometheus",
2532           "description": "",
2533           "fieldConfig": {
2534             "defaults": {
2535               "color": {
2536                 "mode": "palette-classic"
2537               },
2538               "custom": {
2539                 "axisLabel": "",
2540                 "axisPlacement": "auto",
2541                 "barAlignment": 0,
2542                 "drawStyle": "line",
2543                 "fillOpacity": 10,
2544                 "gradientMode": "none",
2545                 "hideFrom": {
2546                   "legend": false,
2547                   "tooltip": false,
2548                   "viz": false
2549                 },
2550                 "lineInterpolation": "linear",
2551                 "lineWidth": 1,
2552                 "pointSize": 5,
2553                 "scaleDistribution": {
2554                   "type": "linear"
2555                 },
2556                 "showPoints": "never",
2557                 "spanNulls": true,
2558                 "stacking": {
2559                   "group": "A",
2560                   "mode": "none"
2561                 },
2562                 "thresholdsStyle": {
2563                   "mode": "off"
2564                 }
2565               },
2566               "links": [],
2567               "mappings": [],
2568               "min": 0,
2569               "thresholds": {
2570                 "mode": "absolute",
2571                 "steps": [
2572                   {
2573                     "color": "green",
2574                     "value": null
2575                   },
2576                   {
2577                     "color": "red",
2578                     "value": 80
2579                   }
2580                 ]
2581               },
2582               "unit": "short"
2583             },
2584             "overrides": []
2585           },
2586           "gridPos": {
2587             "h": 9,
2588             "w": 12,
2589             "x": 12,
2590             "y": 26
2591           },
2592           "id": 92,
2593           "options": {
2594             "legend": {
2595               "calcs": [
2596                 "mean",
2597                 "lastNotNull",
2598                 "max"
2599               ],
2600               "displayMode": "table",
2601               "placement": "bottom"
2602             },
2603             "tooltip": {
2604               "mode": "single"
2605             }
2606           },
2607           "pluginVersion": "8.1.3",
2608           "targets": [
2609             {
2610               "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2611               "hide": false,
2612               "instant": false,
2613               "interval": "",
2614               "legendFormat": "{{instance}}",
2615               "refId": "A"
2616             }
2617           ],
2618           "timeFrom": null,
2619           "timeShift": null,
2620           "title": "Zookeeper connections per sec",
2621           "type": "timeseries"
2622         },
2623         {
2624           "datasource": "Prometheus",
2625           "description": "",
2626           "fieldConfig": {
2627             "defaults": {
2628               "color": {
2629                 "mode": "palette-classic"
2630               },
2631               "custom": {
2632                 "axisLabel": "",
2633                 "axisPlacement": "auto",
2634                 "barAlignment": 0,
2635                 "drawStyle": "line",
2636                 "fillOpacity": 10,
2637                 "gradientMode": "none",
2638                 "hideFrom": {
2639                   "legend": false,
2640                   "tooltip": false,
2641                   "viz": false
2642                 },
2643                 "lineInterpolation": "linear",
2644                 "lineWidth": 1,
2645                 "pointSize": 5,
2646                 "scaleDistribution": {
2647                   "type": "linear"
2648                 },
2649                 "showPoints": "never",
2650                 "spanNulls": true,
2651                 "stacking": {
2652                   "group": "A",
2653                   "mode": "none"
2654                 },
2655                 "thresholdsStyle": {
2656                   "mode": "off"
2657                 }
2658               },
2659               "links": [],
2660               "mappings": [],
2661               "min": 0,
2662               "thresholds": {
2663                 "mode": "absolute",
2664                 "steps": [
2665                   {
2666                     "color": "green",
2667                     "value": null
2668                   },
2669                   {
2670                     "color": "red",
2671                     "value": 80
2672                   }
2673                 ]
2674               },
2675               "unit": "short"
2676             },
2677             "overrides": []
2678           },
2679           "gridPos": {
2680             "h": 9,
2681             "w": 8,
2682             "x": 0,
2683             "y": 35
2684           },
2685           "id": 89,
2686           "options": {
2687             "legend": {
2688               "calcs": [
2689                 "mean",
2690                 "lastNotNull",
2691                 "max"
2692               ],
2693               "displayMode": "table",
2694               "placement": "bottom"
2695             },
2696             "tooltip": {
2697               "mode": "single"
2698             }
2699           },
2700           "pluginVersion": "8.1.3",
2701           "targets": [
2702             {
2703               "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2704               "hide": false,
2705               "interval": "",
2706               "legendFormat": "{{instance}}",
2707               "refId": "A"
2708             }
2709           ],
2710           "timeFrom": null,
2711           "timeShift": null,
2712           "title": "Zookeeper expired connections per sec",
2713           "type": "timeseries"
2714         },
2715         {
2716           "datasource": "Prometheus",
2717           "description": "",
2718           "fieldConfig": {
2719             "defaults": {
2720               "color": {
2721                 "mode": "palette-classic"
2722               },
2723               "custom": {
2724                 "axisLabel": "",
2725                 "axisPlacement": "auto",
2726                 "barAlignment": 0,
2727                 "drawStyle": "line",
2728                 "fillOpacity": 10,
2729                 "gradientMode": "none",
2730                 "hideFrom": {
2731                   "legend": false,
2732                   "tooltip": false,
2733                   "viz": false
2734                 },
2735                 "lineInterpolation": "linear",
2736                 "lineWidth": 1,
2737                 "pointSize": 5,
2738                 "scaleDistribution": {
2739                   "type": "linear"
2740                 },
2741                 "showPoints": "never",
2742                 "spanNulls": true,
2743                 "stacking": {
2744                   "group": "A",
2745                   "mode": "none"
2746                 },
2747                 "thresholdsStyle": {
2748                   "mode": "off"
2749                 }
2750               },
2751               "links": [],
2752               "mappings": [],
2753               "min": 0,
2754               "thresholds": {
2755                 "mode": "absolute",
2756                 "steps": [
2757                   {
2758                     "color": "green",
2759                     "value": null
2760                   },
2761                   {
2762                     "color": "red",
2763                     "value": 80
2764                   }
2765                 ]
2766               },
2767               "unit": "short"
2768             },
2769             "overrides": []
2770           },
2771           "gridPos": {
2772             "h": 9,
2773             "w": 8,
2774             "x": 8,
2775             "y": 35
2776           },
2777           "id": 90,
2778           "options": {
2779             "legend": {
2780               "calcs": [
2781                 "mean",
2782                 "lastNotNull",
2783                 "max"
2784               ],
2785               "displayMode": "table",
2786               "placement": "bottom"
2787             },
2788             "tooltip": {
2789               "mode": "single"
2790             }
2791           },
2792           "pluginVersion": "8.1.3",
2793           "targets": [
2794             {
2795               "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2796               "hide": false,
2797               "interval": "",
2798               "legendFormat": "{{instance}}",
2799               "refId": "A"
2800             }
2801           ],
2802           "timeFrom": null,
2803           "timeShift": null,
2804           "title": "Zookeeper disconnect per sec",
2805           "type": "timeseries"
2806         },
2807         {
2808           "datasource": "Prometheus",
2809           "description": "",
2810           "fieldConfig": {
2811             "defaults": {
2812               "color": {
2813                 "mode": "palette-classic"
2814               },
2815               "custom": {
2816                 "axisLabel": "",
2817                 "axisPlacement": "auto",
2818                 "barAlignment": 0,
2819                 "drawStyle": "line",
2820                 "fillOpacity": 10,
2821                 "gradientMode": "none",
2822                 "hideFrom": {
2823                   "legend": false,
2824                   "tooltip": false,
2825                   "viz": false
2826                 },
2827                 "lineInterpolation": "linear",
2828                 "lineWidth": 1,
2829                 "pointSize": 5,
2830                 "scaleDistribution": {
2831                   "type": "linear"
2832                 },
2833                 "showPoints": "never",
2834                 "spanNulls": true,
2835                 "stacking": {
2836                   "group": "A",
2837                   "mode": "none"
2838                 },
2839                 "thresholdsStyle": {
2840                   "mode": "off"
2841                 }
2842               },
2843               "links": [],
2844               "mappings": [],
2845               "min": 0,
2846               "thresholds": {
2847                 "mode": "absolute",
2848                 "steps": [
2849                   {
2850                     "color": "green",
2851                     "value": null
2852                   },
2853                   {
2854                     "color": "red",
2855                     "value": 80
2856                   }
2857                 ]
2858               },
2859               "unit": "short"
2860             },
2861             "overrides": []
2862           },
2863           "gridPos": {
2864             "h": 9,
2865             "w": 8,
2866             "x": 16,
2867             "y": 35
2868           },
2869           "id": 91,
2870           "options": {
2871             "legend": {
2872               "calcs": [
2873                 "mean",
2874                 "lastNotNull",
2875                 "max"
2876               ],
2877               "displayMode": "table",
2878               "placement": "bottom"
2879             },
2880             "tooltip": {
2881               "mode": "single"
2882             }
2883           },
2884           "pluginVersion": "8.1.3",
2885           "targets": [
2886             {
2887               "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2888               "hide": false,
2889               "interval": "",
2890               "legendFormat": "{{instance}}",
2891               "refId": "A"
2892             }
2893           ],
2894           "timeFrom": null,
2895           "timeShift": null,
2896           "title": "Zookeeper auth failures per sec",
2897           "type": "timeseries"
2898         }
2899       ],
2900       "title": "Zookeeper",
2901       "type": "row"
2902     },
2903     {
2904       "collapsed": true,
2905       "datasource": null,
2906       "gridPos": {
2907         "h": 1,
2908         "w": 24,
2909         "x": 0,
2910         "y": 30
2911       },
2912       "id": 82,
2913       "panels": [
2914         {
2915           "datasource": "Prometheus",
2916           "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2917           "fieldConfig": {
2918             "defaults": {
2919               "color": {
2920                 "mode": "palette-classic"
2921               },
2922               "custom": {
2923                 "axisLabel": "",
2924                 "axisPlacement": "auto",
2925                 "barAlignment": 0,
2926                 "drawStyle": "line",
2927                 "fillOpacity": 10,
2928                 "gradientMode": "none",
2929                 "hideFrom": {
2930                   "legend": false,
2931                   "tooltip": false,
2932                   "viz": false
2933                 },
2934                 "lineInterpolation": "linear",
2935                 "lineWidth": 1,
2936                 "pointSize": 5,
2937                 "scaleDistribution": {
2938                   "type": "linear"
2939                 },
2940                 "showPoints": "never",
2941                 "spanNulls": true,
2942                 "stacking": {
2943                   "group": "A",
2944                   "mode": "none"
2945                 },
2946                 "thresholdsStyle": {
2947                   "mode": "off"
2948                 }
2949               },
2950               "links": [],
2951               "mappings": [],
2952               "min": 0,
2953               "thresholds": {
2954                 "mode": "absolute",
2955                 "steps": [
2956                   {
2957                     "color": "green",
2958                     "value": null
2959                   },
2960                   {
2961                     "color": "red",
2962                     "value": 80
2963                   }
2964                 ]
2965               },
2966               "unit": "ops"
2967             },
2968             "overrides": []
2969           },
2970           "gridPos": {
2971             "h": 9,
2972             "w": 12,
2973             "x": 0,
2974             "y": 27
2975           },
2976           "id": 80,
2977           "options": {
2978             "legend": {
2979               "calcs": [
2980                 "mean",
2981                 "lastNotNull",
2982                 "max"
2983               ],
2984               "displayMode": "table",
2985               "placement": "bottom"
2986             },
2987             "tooltip": {
2988               "mode": "single"
2989             }
2990           },
2991           "pluginVersion": "8.1.3",
2992           "targets": [
2993             {
2994               "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2995               "interval": "",
2996               "legendFormat": "{{instance}}",
2997               "refId": "A"
2998             }
2999           ],
3000           "timeFrom": null,
3001           "timeShift": null,
3002           "title": "IsrShrinks per Sec",
3003           "type": "timeseries"
3004         },
3005         {
3006           "datasource": "Prometheus",
3007           "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
3008           "fieldConfig": {
3009             "defaults": {
3010               "color": {
3011                 "mode": "palette-classic"
3012               },
3013               "custom": {
3014                 "axisLabel": "",
3015                 "axisPlacement": "auto",
3016                 "barAlignment": 0,
3017                 "drawStyle": "line",
3018                 "fillOpacity": 10,
3019                 "gradientMode": "none",
3020                 "hideFrom": {
3021                   "legend": false,
3022                   "tooltip": false,
3023                   "viz": false
3024                 },
3025                 "lineInterpolation": "linear",
3026                 "lineWidth": 1,
3027                 "pointSize": 5,
3028                 "scaleDistribution": {
3029                   "type": "linear"
3030                 },
3031                 "showPoints": "never",
3032                 "spanNulls": true,
3033                 "stacking": {
3034                   "group": "A",
3035                   "mode": "none"
3036                 },
3037                 "thresholdsStyle": {
3038                   "mode": "off"
3039                 }
3040               },
3041               "links": [],
3042               "mappings": [],
3043               "min": 0,
3044               "thresholds": {
3045                 "mode": "absolute",
3046                 "steps": [
3047                   {
3048                     "color": "green",
3049                     "value": null
3050                   },
3051                   {
3052                     "color": "red",
3053                     "value": 80
3054                   }
3055                 ]
3056               },
3057               "unit": "ops"
3058             },
3059             "overrides": []
3060           },
3061           "gridPos": {
3062             "h": 9,
3063             "w": 12,
3064             "x": 12,
3065             "y": 27
3066           },
3067           "id": 83,
3068           "options": {
3069             "legend": {
3070               "calcs": [
3071                 "mean",
3072                 "lastNotNull",
3073                 "max"
3074               ],
3075               "displayMode": "table",
3076               "placement": "bottom"
3077             },
3078             "tooltip": {
3079               "mode": "single"
3080             }
3081           },
3082           "pluginVersion": "8.1.3",
3083           "targets": [
3084             {
3085               "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3086               "hide": false,
3087               "interval": "",
3088               "legendFormat": "{{instance}}",
3089               "refId": "A"
3090             }
3091           ],
3092           "timeFrom": null,
3093           "timeShift": null,
3094           "title": "IsrExpands per Sec",
3095           "type": "timeseries"
3096         }
3097       ],
3098       "title": "Isr Shrinks / Expands",
3099       "type": "row"
3100     },
3101     {
3102       "collapsed": true,
3103       "datasource": null,
3104       "gridPos": {
3105         "h": 1,
3106         "w": 24,
3107         "x": 0,
3108         "y": 31
3109       },
3110       "id": 53,
3111       "panels": [
3112         {
3113           "datasource": "Prometheus",
3114           "fieldConfig": {
3115             "defaults": {
3116               "color": {
3117                 "mode": "palette-classic"
3118               },
3119               "custom": {
3120                 "axisLabel": "",
3121                 "axisPlacement": "auto",
3122                 "barAlignment": 0,
3123                 "drawStyle": "line",
3124                 "fillOpacity": 10,
3125                 "gradientMode": "none",
3126                 "hideFrom": {
3127                   "legend": false,
3128                   "tooltip": false,
3129                   "viz": false
3130                 },
3131                 "lineInterpolation": "linear",
3132                 "lineWidth": 1,
3133                 "pointSize": 5,
3134                 "scaleDistribution": {
3135                   "type": "linear"
3136                 },
3137                 "showPoints": "never",
3138                 "spanNulls": true,
3139                 "stacking": {
3140                   "group": "A",
3141                   "mode": "none"
3142                 },
3143                 "thresholdsStyle": {
3144                   "mode": "off"
3145                 }
3146               },
3147               "links": [],
3148               "mappings": [],
3149               "thresholds": {
3150                 "mode": "absolute",
3151                 "steps": [
3152                   {
3153                     "color": "green",
3154                     "value": null
3155                   },
3156                   {
3157                     "color": "red",
3158                     "value": 80
3159                   }
3160                 ]
3161               },
3162               "unit": "bytes"
3163             },
3164             "overrides": []
3165           },
3166           "gridPos": {
3167             "h": 9,
3168             "w": 12,
3169             "x": 0,
3170             "y": 28
3171           },
3172           "id": 55,
3173           "options": {
3174             "legend": {
3175               "calcs": [
3176                 "mean",
3177                 "lastNotNull",
3178                 "max"
3179               ],
3180               "displayMode": "table",
3181               "placement": "bottom"
3182             },
3183             "tooltip": {
3184               "mode": "single"
3185             }
3186           },
3187           "pluginVersion": "8.1.3",
3188           "targets": [
3189             {
3190               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3191               "legendFormat": "{{topic}}",
3192               "refId": "A"
3193             }
3194           ],
3195           "timeFrom": null,
3196           "timeShift": null,
3197           "title": "Log size per Topic",
3198           "type": "timeseries"
3199         },
3200         {
3201           "datasource": "Prometheus",
3202           "fieldConfig": {
3203             "defaults": {
3204               "color": {
3205                 "mode": "palette-classic"
3206               },
3207               "custom": {
3208                 "axisLabel": "",
3209                 "axisPlacement": "auto",
3210                 "barAlignment": 0,
3211                 "drawStyle": "line",
3212                 "fillOpacity": 10,
3213                 "gradientMode": "none",
3214                 "hideFrom": {
3215                   "legend": false,
3216                   "tooltip": false,
3217                   "viz": false
3218                 },
3219                 "lineInterpolation": "linear",
3220                 "lineWidth": 1,
3221                 "pointSize": 5,
3222                 "scaleDistribution": {
3223                   "type": "linear"
3224                 },
3225                 "showPoints": "never",
3226                 "spanNulls": true,
3227                 "stacking": {
3228                   "group": "A",
3229                   "mode": "none"
3230                 },
3231                 "thresholdsStyle": {
3232                   "mode": "off"
3233                 }
3234               },
3235               "links": [],
3236               "mappings": [],
3237               "thresholds": {
3238                 "mode": "absolute",
3239                 "steps": [
3240                   {
3241                     "color": "green",
3242                     "value": null
3243                   },
3244                   {
3245                     "color": "red",
3246                     "value": 80
3247                   }
3248                 ]
3249               },
3250               "unit": "bytes"
3251             },
3252             "overrides": []
3253           },
3254           "gridPos": {
3255             "h": 9,
3256             "w": 12,
3257             "x": 12,
3258             "y": 28
3259           },
3260           "id": 56,
3261           "options": {
3262             "legend": {
3263               "calcs": [
3264                 "mean",
3265                 "lastNotNull",
3266                 "max"
3267               ],
3268               "displayMode": "table",
3269               "placement": "bottom"
3270             },
3271             "tooltip": {
3272               "mode": "single"
3273             }
3274           },
3275           "pluginVersion": "8.1.3",
3276           "targets": [
3277             {
3278               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3279               "legendFormat": "{{instance}}",
3280               "refId": "A"
3281             }
3282           ],
3283           "timeFrom": null,
3284           "timeShift": null,
3285           "title": "Log size per Broker",
3286           "type": "timeseries"
3287         }
3288       ],
3289       "title": "Logs size",
3290       "type": "row"
3291     },
3292     {
3293       "collapsed": true,
3294       "datasource": null,
3295       "gridPos": {
3296         "h": 1,
3297         "w": 24,
3298         "x": 0,
3299         "y": 32
3300       },
3301       "id": 58,
3302       "panels": [
3303         {
3304           "datasource": "Prometheus",
3305           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3306           "fieldConfig": {
3307             "defaults": {
3308               "color": {
3309                 "mode": "palette-classic"
3310               },
3311               "custom": {
3312                 "axisLabel": "",
3313                 "axisPlacement": "auto",
3314                 "barAlignment": 0,
3315                 "drawStyle": "line",
3316                 "fillOpacity": 10,
3317                 "gradientMode": "none",
3318                 "hideFrom": {
3319                   "legend": false,
3320                   "tooltip": false,
3321                   "viz": false
3322                 },
3323                 "lineInterpolation": "linear",
3324                 "lineWidth": 1,
3325                 "pointSize": 5,
3326                 "scaleDistribution": {
3327                   "type": "linear"
3328                 },
3329                 "showPoints": "never",
3330                 "spanNulls": true,
3331                 "stacking": {
3332                   "group": "A",
3333                   "mode": "none"
3334                 },
3335                 "thresholdsStyle": {
3336                   "mode": "off"
3337                 }
3338               },
3339               "links": [],
3340               "mappings": [],
3341               "thresholds": {
3342                 "mode": "absolute",
3343                 "steps": [
3344                   {
3345                     "color": "green",
3346                     "value": null
3347                   },
3348                   {
3349                     "color": "red",
3350                     "value": 80
3351                   }
3352                 ]
3353               },
3354               "unit": "ms"
3355             },
3356             "overrides": []
3357           },
3358           "gridPos": {
3359             "h": 9,
3360             "w": 12,
3361             "x": 0,
3362             "y": 29
3363           },
3364           "id": 60,
3365           "options": {
3366             "legend": {
3367               "calcs": [
3368                 "mean",
3369                 "lastNotNull",
3370                 "max"
3371               ],
3372               "displayMode": "list",
3373               "placement": "bottom"
3374             },
3375             "tooltip": {
3376               "mode": "single"
3377             }
3378           },
3379           "pluginVersion": "8.1.3",
3380           "targets": [
3381             {
3382               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3383               "hide": false,
3384               "legendFormat": "{{instance}} - {{quantile}}",
3385               "refId": "A"
3386             }
3387           ],
3388           "timeFrom": null,
3389           "timeShift": null,
3390           "title": "Producer - RequestQueueTimeMs",
3391           "type": "timeseries"
3392         },
3393         {
3394           "datasource": "Prometheus",
3395           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3396           "fieldConfig": {
3397             "defaults": {
3398               "color": {
3399                 "mode": "palette-classic"
3400               },
3401               "custom": {
3402                 "axisLabel": "",
3403                 "axisPlacement": "auto",
3404                 "barAlignment": 0,
3405                 "drawStyle": "line",
3406                 "fillOpacity": 10,
3407                 "gradientMode": "none",
3408                 "hideFrom": {
3409                   "legend": false,
3410                   "tooltip": false,
3411                   "viz": false
3412                 },
3413                 "lineInterpolation": "linear",
3414                 "lineWidth": 1,
3415                 "pointSize": 5,
3416                 "scaleDistribution": {
3417                   "type": "linear"
3418                 },
3419                 "showPoints": "never",
3420                 "spanNulls": true,
3421                 "stacking": {
3422                   "group": "A",
3423                   "mode": "none"
3424                 },
3425                 "thresholdsStyle": {
3426                   "mode": "off"
3427                 }
3428               },
3429               "links": [],
3430               "mappings": [],
3431               "thresholds": {
3432                 "mode": "absolute",
3433                 "steps": [
3434                   {
3435                     "color": "green",
3436                     "value": null
3437                   },
3438                   {
3439                     "color": "red",
3440                     "value": 80
3441                   }
3442                 ]
3443               },
3444               "unit": "ms"
3445             },
3446             "overrides": []
3447           },
3448           "gridPos": {
3449             "h": 9,
3450             "w": 12,
3451             "x": 12,
3452             "y": 29
3453           },
3454           "id": 61,
3455           "options": {
3456             "legend": {
3457               "calcs": [
3458                 "mean",
3459                 "lastNotNull",
3460                 "max"
3461               ],
3462               "displayMode": "list",
3463               "placement": "bottom"
3464             },
3465             "tooltip": {
3466               "mode": "single"
3467             }
3468           },
3469           "pluginVersion": "8.1.3",
3470           "targets": [
3471             {
3472               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3473               "hide": false,
3474               "legendFormat": "{{instance}} - {{quantile}}",
3475               "refId": "A"
3476             }
3477           ],
3478           "timeFrom": null,
3479           "timeShift": null,
3480           "title": "Producer - LocalTimeMs",
3481           "type": "timeseries"
3482         },
3483         {
3484           "datasource": "Prometheus",
3485           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3486           "fieldConfig": {
3487             "defaults": {
3488               "color": {
3489                 "mode": "palette-classic"
3490               },
3491               "custom": {
3492                 "axisLabel": "",
3493                 "axisPlacement": "auto",
3494                 "barAlignment": 0,
3495                 "drawStyle": "line",
3496                 "fillOpacity": 10,
3497                 "gradientMode": "none",
3498                 "hideFrom": {
3499                   "legend": false,
3500                   "tooltip": false,
3501                   "viz": false
3502                 },
3503                 "lineInterpolation": "linear",
3504                 "lineWidth": 1,
3505                 "pointSize": 5,
3506                 "scaleDistribution": {
3507                   "type": "linear"
3508                 },
3509                 "showPoints": "never",
3510                 "spanNulls": true,
3511                 "stacking": {
3512                   "group": "A",
3513                   "mode": "none"
3514                 },
3515                 "thresholdsStyle": {
3516                   "mode": "off"
3517                 }
3518               },
3519               "links": [],
3520               "mappings": [],
3521               "thresholds": {
3522                 "mode": "absolute",
3523                 "steps": [
3524                   {
3525                     "color": "green",
3526                     "value": null
3527                   },
3528                   {
3529                     "color": "red",
3530                     "value": 80
3531                   }
3532                 ]
3533               },
3534               "unit": "ms"
3535             },
3536             "overrides": []
3537           },
3538           "gridPos": {
3539             "h": 10,
3540             "w": 8,
3541             "x": 0,
3542             "y": 38
3543           },
3544           "id": 62,
3545           "options": {
3546             "legend": {
3547               "calcs": [
3548                 "mean",
3549                 "lastNotNull",
3550                 "max"
3551               ],
3552               "displayMode": "table",
3553               "placement": "bottom"
3554             },
3555             "tooltip": {
3556               "mode": "single"
3557             }
3558           },
3559           "pluginVersion": "8.1.3",
3560           "targets": [
3561             {
3562               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3563               "hide": false,
3564               "legendFormat": "{{instance}} - {{quantile}}",
3565               "refId": "A"
3566             }
3567           ],
3568           "timeFrom": null,
3569           "timeShift": null,
3570           "title": "Producer - RemoteTimeMs",
3571           "type": "timeseries"
3572         },
3573         {
3574           "datasource": "Prometheus",
3575           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3576           "fieldConfig": {
3577             "defaults": {
3578               "color": {
3579                 "mode": "palette-classic"
3580               },
3581               "custom": {
3582                 "axisLabel": "",
3583                 "axisPlacement": "auto",
3584                 "barAlignment": 0,
3585                 "drawStyle": "line",
3586                 "fillOpacity": 10,
3587                 "gradientMode": "none",
3588                 "hideFrom": {
3589                   "legend": false,
3590                   "tooltip": false,
3591                   "viz": false
3592                 },
3593                 "lineInterpolation": "linear",
3594                 "lineWidth": 1,
3595                 "pointSize": 5,
3596                 "scaleDistribution": {
3597                   "type": "linear"
3598                 },
3599                 "showPoints": "never",
3600                 "spanNulls": true,
3601                 "stacking": {
3602                   "group": "A",
3603                   "mode": "none"
3604                 },
3605                 "thresholdsStyle": {
3606                   "mode": "off"
3607                 }
3608               },
3609               "links": [],
3610               "mappings": [],
3611               "thresholds": {
3612                 "mode": "absolute",
3613                 "steps": [
3614                   {
3615                     "color": "green",
3616                     "value": null
3617                   },
3618                   {
3619                     "color": "red",
3620                     "value": 80
3621                   }
3622                 ]
3623               },
3624               "unit": "ms"
3625             },
3626             "overrides": []
3627           },
3628           "gridPos": {
3629             "h": 10,
3630             "w": 8,
3631             "x": 8,
3632             "y": 38
3633           },
3634           "id": 63,
3635           "options": {
3636             "legend": {
3637               "calcs": [
3638                 "mean",
3639                 "lastNotNull",
3640                 "max"
3641               ],
3642               "displayMode": "table",
3643               "placement": "bottom"
3644             },
3645             "tooltip": {
3646               "mode": "single"
3647             }
3648           },
3649           "pluginVersion": "8.1.3",
3650           "targets": [
3651             {
3652               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3653               "hide": false,
3654               "legendFormat": "{{instance}} - {{quantile}}",
3655               "refId": "A"
3656             }
3657           ],
3658           "timeFrom": null,
3659           "timeShift": null,
3660           "title": "Producer - ResponseQueueTimeMs",
3661           "type": "timeseries"
3662         },
3663         {
3664           "datasource": "Prometheus",
3665           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3666           "fieldConfig": {
3667             "defaults": {
3668               "color": {
3669                 "mode": "palette-classic"
3670               },
3671               "custom": {
3672                 "axisLabel": "",
3673                 "axisPlacement": "auto",
3674                 "barAlignment": 0,
3675                 "drawStyle": "line",
3676                 "fillOpacity": 10,
3677                 "gradientMode": "none",
3678                 "hideFrom": {
3679                   "legend": false,
3680                   "tooltip": false,
3681                   "viz": false
3682                 },
3683                 "lineInterpolation": "linear",
3684                 "lineWidth": 1,
3685                 "pointSize": 5,
3686                 "scaleDistribution": {
3687                   "type": "linear"
3688                 },
3689                 "showPoints": "never",
3690                 "spanNulls": true,
3691                 "stacking": {
3692                   "group": "A",
3693                   "mode": "none"
3694                 },
3695                 "thresholdsStyle": {
3696                   "mode": "off"
3697                 }
3698               },
3699               "links": [],
3700               "mappings": [],
3701               "min": 0,
3702               "thresholds": {
3703                 "mode": "absolute",
3704                 "steps": [
3705                   {
3706                     "color": "green",
3707                     "value": null
3708                   },
3709                   {
3710                     "color": "red",
3711                     "value": 80
3712                   }
3713                 ]
3714               },
3715               "unit": "ms"
3716             },
3717             "overrides": []
3718           },
3719           "gridPos": {
3720             "h": 10,
3721             "w": 8,
3722             "x": 16,
3723             "y": 38
3724           },
3725           "id": 64,
3726           "options": {
3727             "legend": {
3728               "calcs": [
3729                 "mean",
3730                 "lastNotNull",
3731                 "max"
3732               ],
3733               "displayMode": "table",
3734               "placement": "bottom"
3735             },
3736             "tooltip": {
3737               "mode": "single"
3738             }
3739           },
3740           "pluginVersion": "8.1.3",
3741           "targets": [
3742             {
3743               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3744               "hide": false,
3745               "legendFormat": "{{instance}} - {{quantile}}",
3746               "refId": "A"
3747             }
3748           ],
3749           "timeFrom": null,
3750           "timeShift": null,
3751           "title": "Producer - ResponseSendTimeMs",
3752           "type": "timeseries"
3753         }
3754       ],
3755       "title": "Producer Performance",
3756       "type": "row"
3757     },
3758     {
3759       "collapsed": true,
3760       "datasource": null,
3761       "gridPos": {
3762         "h": 1,
3763         "w": 24,
3764         "x": 0,
3765         "y": 33
3766       },
3767       "id": 68,
3768       "panels": [
3769         {
3770           "datasource": "Prometheus",
3771           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3772           "fieldConfig": {
3773             "defaults": {
3774               "color": {
3775                 "mode": "palette-classic"
3776               },
3777               "custom": {
3778                 "axisLabel": "",
3779                 "axisPlacement": "auto",
3780                 "barAlignment": 0,
3781                 "drawStyle": "line",
3782                 "fillOpacity": 10,
3783                 "gradientMode": "none",
3784                 "hideFrom": {
3785                   "legend": false,
3786                   "tooltip": false,
3787                   "viz": false
3788                 },
3789                 "lineInterpolation": "linear",
3790                 "lineWidth": 1,
3791                 "pointSize": 5,
3792                 "scaleDistribution": {
3793                   "type": "linear"
3794                 },
3795                 "showPoints": "never",
3796                 "spanNulls": true,
3797                 "stacking": {
3798                   "group": "A",
3799                   "mode": "none"
3800                 },
3801                 "thresholdsStyle": {
3802                   "mode": "off"
3803                 }
3804               },
3805               "links": [],
3806               "mappings": [],
3807               "min": 0,
3808               "thresholds": {
3809                 "mode": "absolute",
3810                 "steps": [
3811                   {
3812                     "color": "green",
3813                     "value": null
3814                   },
3815                   {
3816                     "color": "red",
3817                     "value": 80
3818                   }
3819                 ]
3820               },
3821               "unit": "ms"
3822             },
3823             "overrides": []
3824           },
3825           "gridPos": {
3826             "h": 9,
3827             "w": 12,
3828             "x": 0,
3829             "y": 30
3830           },
3831           "id": 69,
3832           "options": {
3833             "legend": {
3834               "calcs": [
3835                 "mean",
3836                 "lastNotNull",
3837                 "max"
3838               ],
3839               "displayMode": "list",
3840               "placement": "bottom"
3841             },
3842             "tooltip": {
3843               "mode": "single"
3844             }
3845           },
3846           "pluginVersion": "8.1.3",
3847           "targets": [
3848             {
3849               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3850               "hide": false,
3851               "legendFormat": "{{instance}} - {{quantile}}",
3852               "refId": "A"
3853             }
3854           ],
3855           "timeFrom": null,
3856           "timeShift": null,
3857           "title": "Consumer - RequestQueueTimeMs",
3858           "type": "timeseries"
3859         },
3860         {
3861           "datasource": "Prometheus",
3862           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3863           "fieldConfig": {
3864             "defaults": {
3865               "color": {
3866                 "mode": "palette-classic"
3867               },
3868               "custom": {
3869                 "axisLabel": "",
3870                 "axisPlacement": "auto",
3871                 "barAlignment": 0,
3872                 "drawStyle": "line",
3873                 "fillOpacity": 10,
3874                 "gradientMode": "none",
3875                 "hideFrom": {
3876                   "legend": false,
3877                   "tooltip": false,
3878                   "viz": false
3879                 },
3880                 "lineInterpolation": "linear",
3881                 "lineWidth": 1,
3882                 "pointSize": 5,
3883                 "scaleDistribution": {
3884                   "type": "linear"
3885                 },
3886                 "showPoints": "never",
3887                 "spanNulls": true,
3888                 "stacking": {
3889                   "group": "A",
3890                   "mode": "none"
3891                 },
3892                 "thresholdsStyle": {
3893                   "mode": "off"
3894                 }
3895               },
3896               "links": [],
3897               "mappings": [],
3898               "thresholds": {
3899                 "mode": "absolute",
3900                 "steps": [
3901                   {
3902                     "color": "green",
3903                     "value": null
3904                   },
3905                   {
3906                     "color": "red",
3907                     "value": 80
3908                   }
3909                 ]
3910               },
3911               "unit": "ms"
3912             },
3913             "overrides": []
3914           },
3915           "gridPos": {
3916             "h": 9,
3917             "w": 12,
3918             "x": 12,
3919             "y": 30
3920           },
3921           "id": 70,
3922           "options": {
3923             "legend": {
3924               "calcs": [
3925                 "mean",
3926                 "lastNotNull",
3927                 "max"
3928               ],
3929               "displayMode": "list",
3930               "placement": "bottom"
3931             },
3932             "tooltip": {
3933               "mode": "single"
3934             }
3935           },
3936           "pluginVersion": "8.1.3",
3937           "targets": [
3938             {
3939               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3940               "hide": false,
3941               "legendFormat": "{{instance}} - {{quantile}}",
3942               "refId": "A"
3943             }
3944           ],
3945           "timeFrom": null,
3946           "timeShift": null,
3947           "title": "Consumer - LocalTimeMs",
3948           "type": "timeseries"
3949         },
3950         {
3951           "datasource": "Prometheus",
3952           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3953           "fieldConfig": {
3954             "defaults": {
3955               "color": {
3956                 "mode": "palette-classic"
3957               },
3958               "custom": {
3959                 "axisLabel": "",
3960                 "axisPlacement": "auto",
3961                 "barAlignment": 0,
3962                 "drawStyle": "line",
3963                 "fillOpacity": 10,
3964                 "gradientMode": "none",
3965                 "hideFrom": {
3966                   "legend": false,
3967                   "tooltip": false,
3968                   "viz": false
3969                 },
3970                 "lineInterpolation": "linear",
3971                 "lineWidth": 1,
3972                 "pointSize": 5,
3973                 "scaleDistribution": {
3974                   "type": "linear"
3975                 },
3976                 "showPoints": "never",
3977                 "spanNulls": true,
3978                 "stacking": {
3979                   "group": "A",
3980                   "mode": "none"
3981                 },
3982                 "thresholdsStyle": {
3983                   "mode": "off"
3984                 }
3985               },
3986               "links": [],
3987               "mappings": [],
3988               "thresholds": {
3989                 "mode": "absolute",
3990                 "steps": [
3991                   {
3992                     "color": "green",
3993                     "value": null
3994                   },
3995                   {
3996                     "color": "red",
3997                     "value": 80
3998                   }
3999                 ]
4000               },
4001               "unit": "ms"
4002             },
4003             "overrides": []
4004           },
4005           "gridPos": {
4006             "h": 9,
4007             "w": 8,
4008             "x": 0,
4009             "y": 39
4010           },
4011           "id": 71,
4012           "options": {
4013             "legend": {
4014               "calcs": [
4015                 "mean",
4016                 "lastNotNull",
4017                 "max"
4018               ],
4019               "displayMode": "table",
4020               "placement": "bottom"
4021             },
4022             "tooltip": {
4023               "mode": "single"
4024             }
4025           },
4026           "pluginVersion": "8.1.3",
4027           "targets": [
4028             {
4029               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4030               "hide": false,
4031               "legendFormat": "{{instance}} - {{quantile}}",
4032               "refId": "A"
4033             }
4034           ],
4035           "timeFrom": null,
4036           "timeShift": null,
4037           "title": "Consumer - RemoteTimeMs",
4038           "type": "timeseries"
4039         },
4040         {
4041           "datasource": "Prometheus",
4042           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4043           "fieldConfig": {
4044             "defaults": {
4045               "color": {
4046                 "mode": "palette-classic"
4047               },
4048               "custom": {
4049                 "axisLabel": "",
4050                 "axisPlacement": "auto",
4051                 "barAlignment": 0,
4052                 "drawStyle": "line",
4053                 "fillOpacity": 10,
4054                 "gradientMode": "none",
4055                 "hideFrom": {
4056                   "legend": false,
4057                   "tooltip": false,
4058                   "viz": false
4059                 },
4060                 "lineInterpolation": "linear",
4061                 "lineWidth": 1,
4062                 "pointSize": 5,
4063                 "scaleDistribution": {
4064                   "type": "linear"
4065                 },
4066                 "showPoints": "never",
4067                 "spanNulls": true,
4068                 "stacking": {
4069                   "group": "A",
4070                   "mode": "none"
4071                 },
4072                 "thresholdsStyle": {
4073                   "mode": "off"
4074                 }
4075               },
4076               "links": [],
4077               "mappings": [],
4078               "min": 0,
4079               "thresholds": {
4080                 "mode": "absolute",
4081                 "steps": [
4082                   {
4083                     "color": "green",
4084                     "value": null
4085                   },
4086                   {
4087                     "color": "red",
4088                     "value": 80
4089                   }
4090                 ]
4091               },
4092               "unit": "ms"
4093             },
4094             "overrides": []
4095           },
4096           "gridPos": {
4097             "h": 9,
4098             "w": 8,
4099             "x": 8,
4100             "y": 39
4101           },
4102           "id": 72,
4103           "options": {
4104             "legend": {
4105               "calcs": [
4106                 "mean",
4107                 "lastNotNull",
4108                 "max"
4109               ],
4110               "displayMode": "table",
4111               "placement": "bottom"
4112             },
4113             "tooltip": {
4114               "mode": "single"
4115             }
4116           },
4117           "pluginVersion": "8.1.3",
4118           "targets": [
4119             {
4120               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4121               "hide": false,
4122               "legendFormat": "{{instance}} - {{quantile}}",
4123               "refId": "A"
4124             }
4125           ],
4126           "timeFrom": null,
4127           "timeShift": null,
4128           "title": "Consumer - ResponseQueueTimeMs",
4129           "type": "timeseries"
4130         },
4131         {
4132           "datasource": "Prometheus",
4133           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4134           "fieldConfig": {
4135             "defaults": {
4136               "color": {
4137                 "mode": "palette-classic"
4138               },
4139               "custom": {
4140                 "axisLabel": "",
4141                 "axisPlacement": "auto",
4142                 "barAlignment": 0,
4143                 "drawStyle": "line",
4144                 "fillOpacity": 10,
4145                 "gradientMode": "none",
4146                 "hideFrom": {
4147                   "legend": false,
4148                   "tooltip": false,
4149                   "viz": false
4150                 },
4151                 "lineInterpolation": "linear",
4152                 "lineWidth": 1,
4153                 "pointSize": 5,
4154                 "scaleDistribution": {
4155                   "type": "linear"
4156                 },
4157                 "showPoints": "never",
4158                 "spanNulls": true,
4159                 "stacking": {
4160                   "group": "A",
4161                   "mode": "none"
4162                 },
4163                 "thresholdsStyle": {
4164                   "mode": "off"
4165                 }
4166               },
4167               "links": [],
4168               "mappings": [],
4169               "min": 0,
4170               "thresholds": {
4171                 "mode": "absolute",
4172                 "steps": [
4173                   {
4174                     "color": "green",
4175                     "value": null
4176                   },
4177                   {
4178                     "color": "red",
4179                     "value": 80
4180                   }
4181                 ]
4182               },
4183               "unit": "ms"
4184             },
4185             "overrides": []
4186           },
4187           "gridPos": {
4188             "h": 9,
4189             "w": 8,
4190             "x": 16,
4191             "y": 39
4192           },
4193           "id": 73,
4194           "options": {
4195             "legend": {
4196               "calcs": [
4197                 "mean",
4198                 "lastNotNull",
4199                 "max"
4200               ],
4201               "displayMode": "table",
4202               "placement": "bottom"
4203             },
4204             "tooltip": {
4205               "mode": "single"
4206             }
4207           },
4208           "pluginVersion": "8.1.3",
4209           "targets": [
4210             {
4211               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4212               "hide": false,
4213               "legendFormat": "{{instance}} - {{quantile}}",
4214               "refId": "A"
4215             }
4216           ],
4217           "timeFrom": null,
4218           "timeShift": null,
4219           "title": "Consumer - ResponseSendTimeMs",
4220           "type": "timeseries"
4221         }
4222       ],
4223       "title": "Consumer Performance",
4224       "type": "row"
4225     },
4226     {
4227       "collapsed": true,
4228       "datasource": null,
4229       "gridPos": {
4230         "h": 1,
4231         "w": 24,
4232         "x": 0,
4233         "y": 34
4234       },
4235       "id": 66,
4236       "panels": [
4237         {
4238           "datasource": "Prometheus",
4239           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4240           "fieldConfig": {
4241             "defaults": {
4242               "color": {
4243                 "mode": "palette-classic"
4244               },
4245               "custom": {
4246                 "axisLabel": "",
4247                 "axisPlacement": "auto",
4248                 "barAlignment": 0,
4249                 "drawStyle": "line",
4250                 "fillOpacity": 10,
4251                 "gradientMode": "none",
4252                 "hideFrom": {
4253                   "legend": false,
4254                   "tooltip": false,
4255                   "viz": false
4256                 },
4257                 "lineInterpolation": "linear",
4258                 "lineWidth": 1,
4259                 "pointSize": 5,
4260                 "scaleDistribution": {
4261                   "type": "linear"
4262                 },
4263                 "showPoints": "never",
4264                 "spanNulls": true,
4265                 "stacking": {
4266                   "group": "A",
4267                   "mode": "none"
4268                 },
4269                 "thresholdsStyle": {
4270                   "mode": "off"
4271                 }
4272               },
4273               "links": [],
4274               "mappings": [],
4275               "min": 0,
4276               "thresholds": {
4277                 "mode": "absolute",
4278                 "steps": [
4279                   {
4280                     "color": "green",
4281                     "value": null
4282                   },
4283                   {
4284                     "color": "red",
4285                     "value": 80
4286                   }
4287                 ]
4288               },
4289               "unit": "ms"
4290             },
4291             "overrides": []
4292           },
4293           "gridPos": {
4294             "h": 9,
4295             "w": 12,
4296             "x": 0,
4297             "y": 31
4298           },
4299           "id": 74,
4300           "options": {
4301             "legend": {
4302               "calcs": [
4303                 "mean",
4304                 "lastNotNull",
4305                 "max"
4306               ],
4307               "displayMode": "list",
4308               "placement": "bottom"
4309             },
4310             "tooltip": {
4311               "mode": "single"
4312             }
4313           },
4314           "pluginVersion": "8.1.3",
4315           "targets": [
4316             {
4317               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4318               "hide": false,
4319               "legendFormat": "{{instance}} - {{quantile}}",
4320               "refId": "A"
4321             }
4322           ],
4323           "timeFrom": null,
4324           "timeShift": null,
4325           "title": "FetchFollower - RequestQueueTimeMs",
4326           "type": "timeseries"
4327         },
4328         {
4329           "datasource": "Prometheus",
4330           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4331           "fieldConfig": {
4332             "defaults": {
4333               "color": {
4334                 "mode": "palette-classic"
4335               },
4336               "custom": {
4337                 "axisLabel": "",
4338                 "axisPlacement": "auto",
4339                 "barAlignment": 0,
4340                 "drawStyle": "line",
4341                 "fillOpacity": 10,
4342                 "gradientMode": "none",
4343                 "hideFrom": {
4344                   "legend": false,
4345                   "tooltip": false,
4346                   "viz": false
4347                 },
4348                 "lineInterpolation": "linear",
4349                 "lineWidth": 1,
4350                 "pointSize": 5,
4351                 "scaleDistribution": {
4352                   "type": "linear"
4353                 },
4354                 "showPoints": "never",
4355                 "spanNulls": true,
4356                 "stacking": {
4357                   "group": "A",
4358                   "mode": "none"
4359                 },
4360                 "thresholdsStyle": {
4361                   "mode": "off"
4362                 }
4363               },
4364               "links": [],
4365               "mappings": [],
4366               "min": 0,
4367               "thresholds": {
4368                 "mode": "absolute",
4369                 "steps": [
4370                   {
4371                     "color": "green",
4372                     "value": null
4373                   },
4374                   {
4375                     "color": "red",
4376                     "value": 80
4377                   }
4378                 ]
4379               },
4380               "unit": "ms"
4381             },
4382             "overrides": []
4383           },
4384           "gridPos": {
4385             "h": 9,
4386             "w": 12,
4387             "x": 12,
4388             "y": 31
4389           },
4390           "id": 75,
4391           "options": {
4392             "legend": {
4393               "calcs": [
4394                 "mean",
4395                 "lastNotNull",
4396                 "max"
4397               ],
4398               "displayMode": "list",
4399               "placement": "bottom"
4400             },
4401             "tooltip": {
4402               "mode": "single"
4403             }
4404           },
4405           "pluginVersion": "8.1.3",
4406           "targets": [
4407             {
4408               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4409               "hide": false,
4410               "legendFormat": "{{instance}} - {{quantile}}",
4411               "refId": "A"
4412             }
4413           ],
4414           "timeFrom": null,
4415           "timeShift": null,
4416           "title": "FetchFollower - LocalTimeMs",
4417           "type": "timeseries"
4418         },
4419         {
4420           "datasource": "Prometheus",
4421           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4422           "fieldConfig": {
4423             "defaults": {
4424               "color": {
4425                 "mode": "palette-classic"
4426               },
4427               "custom": {
4428                 "axisLabel": "",
4429                 "axisPlacement": "auto",
4430                 "barAlignment": 0,
4431                 "drawStyle": "line",
4432                 "fillOpacity": 10,
4433                 "gradientMode": "none",
4434                 "hideFrom": {
4435                   "legend": false,
4436                   "tooltip": false,
4437                   "viz": false
4438                 },
4439                 "lineInterpolation": "linear",
4440                 "lineWidth": 1,
4441                 "pointSize": 5,
4442                 "scaleDistribution": {
4443                   "type": "linear"
4444                 },
4445                 "showPoints": "never",
4446                 "spanNulls": true,
4447                 "stacking": {
4448                   "group": "A",
4449                   "mode": "none"
4450                 },
4451                 "thresholdsStyle": {
4452                   "mode": "off"
4453                 }
4454               },
4455               "links": [],
4456               "mappings": [],
4457               "thresholds": {
4458                 "mode": "absolute",
4459                 "steps": [
4460                   {
4461                     "color": "green",
4462                     "value": null
4463                   },
4464                   {
4465                     "color": "red",
4466                     "value": 80
4467                   }
4468                 ]
4469               },
4470               "unit": "ms"
4471             },
4472             "overrides": []
4473           },
4474           "gridPos": {
4475             "h": 9,
4476             "w": 8,
4477             "x": 0,
4478             "y": 40
4479           },
4480           "id": 76,
4481           "options": {
4482             "legend": {
4483               "calcs": [
4484                 "mean",
4485                 "lastNotNull",
4486                 "max"
4487               ],
4488               "displayMode": "table",
4489               "placement": "bottom"
4490             },
4491             "tooltip": {
4492               "mode": "single"
4493             }
4494           },
4495           "pluginVersion": "8.1.3",
4496           "targets": [
4497             {
4498               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4499               "hide": false,
4500               "legendFormat": "{{instance}} - {{quantile}}",
4501               "refId": "A"
4502             }
4503           ],
4504           "timeFrom": null,
4505           "timeShift": null,
4506           "title": "FetchFollower - RemoteTimeMs",
4507           "type": "timeseries"
4508         },
4509         {
4510           "datasource": "Prometheus",
4511           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4512           "fieldConfig": {
4513             "defaults": {
4514               "color": {
4515                 "mode": "palette-classic"
4516               },
4517               "custom": {
4518                 "axisLabel": "",
4519                 "axisPlacement": "auto",
4520                 "barAlignment": 0,
4521                 "drawStyle": "line",
4522                 "fillOpacity": 10,
4523                 "gradientMode": "none",
4524                 "hideFrom": {
4525                   "legend": false,
4526                   "tooltip": false,
4527                   "viz": false
4528                 },
4529                 "lineInterpolation": "linear",
4530                 "lineWidth": 1,
4531                 "pointSize": 5,
4532                 "scaleDistribution": {
4533                   "type": "linear"
4534                 },
4535                 "showPoints": "never",
4536                 "spanNulls": true,
4537                 "stacking": {
4538                   "group": "A",
4539                   "mode": "none"
4540                 },
4541                 "thresholdsStyle": {
4542                   "mode": "off"
4543                 }
4544               },
4545               "links": [],
4546               "mappings": [],
4547               "thresholds": {
4548                 "mode": "absolute",
4549                 "steps": [
4550                   {
4551                     "color": "green",
4552                     "value": null
4553                   },
4554                   {
4555                     "color": "red",
4556                     "value": 80
4557                   }
4558                 ]
4559               },
4560               "unit": "ms"
4561             },
4562             "overrides": []
4563           },
4564           "gridPos": {
4565             "h": 9,
4566             "w": 8,
4567             "x": 8,
4568             "y": 40
4569           },
4570           "id": 77,
4571           "options": {
4572             "legend": {
4573               "calcs": [
4574                 "mean",
4575                 "lastNotNull",
4576                 "max"
4577               ],
4578               "displayMode": "table",
4579               "placement": "bottom"
4580             },
4581             "tooltip": {
4582               "mode": "single"
4583             }
4584           },
4585           "pluginVersion": "8.1.3",
4586           "targets": [
4587             {
4588               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4589               "hide": false,
4590               "legendFormat": "{{instance}} - {{quantile}}",
4591               "refId": "A"
4592             }
4593           ],
4594           "timeFrom": null,
4595           "timeShift": null,
4596           "title": "FetchFollower - ResponseQueueTimeMs",
4597           "type": "timeseries"
4598         },
4599         {
4600           "datasource": "Prometheus",
4601           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4602           "fieldConfig": {
4603             "defaults": {
4604               "color": {
4605                 "mode": "palette-classic"
4606               },
4607               "custom": {
4608                 "axisLabel": "",
4609                 "axisPlacement": "auto",
4610                 "barAlignment": 0,
4611                 "drawStyle": "line",
4612                 "fillOpacity": 10,
4613                 "gradientMode": "none",
4614                 "hideFrom": {
4615                   "legend": false,
4616                   "tooltip": false,
4617                   "viz": false
4618                 },
4619                 "lineInterpolation": "linear",
4620                 "lineWidth": 1,
4621                 "pointSize": 5,
4622                 "scaleDistribution": {
4623                   "type": "linear"
4624                 },
4625                 "showPoints": "never",
4626                 "spanNulls": true,
4627                 "stacking": {
4628                   "group": "A",
4629                   "mode": "none"
4630                 },
4631                 "thresholdsStyle": {
4632                   "mode": "off"
4633                 }
4634               },
4635               "links": [],
4636               "mappings": [],
4637               "min": 0,
4638               "thresholds": {
4639                 "mode": "absolute",
4640                 "steps": [
4641                   {
4642                     "color": "green",
4643                     "value": null
4644                   },
4645                   {
4646                     "color": "red",
4647                     "value": 80
4648                   }
4649                 ]
4650               },
4651               "unit": "ms"
4652             },
4653             "overrides": []
4654           },
4655           "gridPos": {
4656             "h": 9,
4657             "w": 8,
4658             "x": 16,
4659             "y": 40
4660           },
4661           "id": 78,
4662           "options": {
4663             "legend": {
4664               "calcs": [
4665                 "mean",
4666                 "lastNotNull",
4667                 "max"
4668               ],
4669               "displayMode": "table",
4670               "placement": "bottom"
4671             },
4672             "tooltip": {
4673               "mode": "single"
4674             }
4675           },
4676           "pluginVersion": "8.1.3",
4677           "targets": [
4678             {
4679               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4680               "hide": false,
4681               "legendFormat": "{{instance}} - {{quantile}}",
4682               "refId": "A"
4683             }
4684           ],
4685           "timeFrom": null,
4686           "timeShift": null,
4687           "title": "FetchFollower - ResponseSendTimeMs",
4688           "type": "timeseries"
4689         }
4690       ],
4691       "title": "Fetch Follower Performance",
4692       "type": "row"
4693     },
4694     {
4695       "collapsed": true,
4696       "datasource": null,
4697       "gridPos": {
4698         "h": 1,
4699         "w": 24,
4700         "x": 0,
4701         "y": 35
4702       },
4703       "id": 102,
4704       "panels": [
4705         {
4706           "datasource": "Prometheus",
4707           "fieldConfig": {
4708             "defaults": {
4709               "color": {
4710                 "mode": "palette-classic"
4711               },
4712               "custom": {
4713                 "axisLabel": "",
4714                 "axisPlacement": "auto",
4715                 "barAlignment": 0,
4716                 "drawStyle": "line",
4717                 "fillOpacity": 10,
4718                 "gradientMode": "none",
4719                 "hideFrom": {
4720                   "legend": false,
4721                   "tooltip": false,
4722                   "viz": false
4723                 },
4724                 "lineInterpolation": "linear",
4725                 "lineWidth": 1,
4726                 "pointSize": 5,
4727                 "scaleDistribution": {
4728                   "type": "linear"
4729                 },
4730                 "showPoints": "never",
4731                 "spanNulls": true,
4732                 "stacking": {
4733                   "group": "A",
4734                   "mode": "none"
4735                 },
4736                 "thresholdsStyle": {
4737                   "mode": "off"
4738                 }
4739               },
4740               "links": [],
4741               "mappings": [],
4742               "thresholds": {
4743                 "mode": "absolute",
4744                 "steps": [
4745                   {
4746                     "color": "green",
4747                     "value": null
4748                   },
4749                   {
4750                     "color": "red",
4751                     "value": 80
4752                   }
4753                 ]
4754               },
4755               "unit": "short"
4756             },
4757             "overrides": []
4758           },
4759           "gridPos": {
4760             "h": 8,
4761             "w": 12,
4762             "x": 0,
4763             "y": 32
4764           },
4765           "id": 98,
4766           "options": {
4767             "legend": {
4768               "calcs": [
4769                 "mean",
4770                 "lastNotNull",
4771                 "max"
4772               ],
4773               "displayMode": "table",
4774               "placement": "bottom"
4775             },
4776             "tooltip": {
4777               "mode": "single"
4778             }
4779           },
4780           "pluginVersion": "8.1.3",
4781           "targets": [
4782             {
4783               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4784               "interval": "",
4785               "legendFormat": "{{listener}}",
4786               "refId": "A"
4787             }
4788           ],
4789           "timeFrom": null,
4790           "timeShift": null,
4791           "title": "Connections count per listener",
4792           "type": "timeseries"
4793         },
4794         {
4795           "datasource": "Prometheus",
4796           "fieldConfig": {
4797             "defaults": {
4798               "color": {
4799                 "mode": "palette-classic"
4800               },
4801               "custom": {
4802                 "axisLabel": "",
4803                 "axisPlacement": "auto",
4804                 "barAlignment": 0,
4805                 "drawStyle": "line",
4806                 "fillOpacity": 10,
4807                 "gradientMode": "none",
4808                 "hideFrom": {
4809                   "legend": false,
4810                   "tooltip": false,
4811                   "viz": false
4812                 },
4813                 "lineInterpolation": "linear",
4814                 "lineWidth": 1,
4815                 "pointSize": 5,
4816                 "scaleDistribution": {
4817                   "type": "linear"
4818                 },
4819                 "showPoints": "never",
4820                 "spanNulls": true,
4821                 "stacking": {
4822                   "group": "A",
4823                   "mode": "none"
4824                 },
4825                 "thresholdsStyle": {
4826                   "mode": "off"
4827                 }
4828               },
4829               "links": [],
4830               "mappings": [],
4831               "thresholds": {
4832                 "mode": "absolute",
4833                 "steps": [
4834                   {
4835                     "color": "green",
4836                     "value": null
4837                   },
4838                   {
4839                     "color": "red",
4840                     "value": 80
4841                   }
4842                 ]
4843               },
4844               "unit": "short"
4845             },
4846             "overrides": []
4847           },
4848           "gridPos": {
4849             "h": 8,
4850             "w": 12,
4851             "x": 12,
4852             "y": 32
4853           },
4854           "id": 100,
4855           "options": {
4856             "legend": {
4857               "calcs": [
4858                 "mean",
4859                 "lastNotNull",
4860                 "max"
4861               ],
4862               "displayMode": "table",
4863               "placement": "bottom"
4864             },
4865             "tooltip": {
4866               "mode": "single"
4867             }
4868           },
4869           "pluginVersion": "8.1.3",
4870           "targets": [
4871             {
4872               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
4873               "interval": "",
4874               "legendFormat": "{{instance}}",
4875               "refId": "A"
4876             }
4877           ],
4878           "timeFrom": null,
4879           "timeShift": null,
4880           "title": "Connections count per broker",
4881           "type": "timeseries"
4882         },
4883         {
4884           "datasource": "Prometheus",
4885           "fieldConfig": {
4886             "defaults": {
4887               "color": {
4888                 "mode": "palette-classic"
4889               },
4890               "custom": {
4891                 "axisLabel": "",
4892                 "axisPlacement": "auto",
4893                 "barAlignment": 0,
4894                 "drawStyle": "line",
4895                 "fillOpacity": 10,
4896                 "gradientMode": "none",
4897                 "hideFrom": {
4898                   "legend": false,
4899                   "tooltip": false,
4900                   "viz": false
4901                 },
4902                 "lineInterpolation": "linear",
4903                 "lineWidth": 1,
4904                 "pointSize": 5,
4905                 "scaleDistribution": {
4906                   "type": "linear"
4907                 },
4908                 "showPoints": "never",
4909                 "spanNulls": true,
4910                 "stacking": {
4911                   "group": "A",
4912                   "mode": "none"
4913                 },
4914                 "thresholdsStyle": {
4915                   "mode": "off"
4916                 }
4917               },
4918               "links": [],
4919               "mappings": [],
4920               "thresholds": {
4921                 "mode": "absolute",
4922                 "steps": [
4923                   {
4924                     "color": "green",
4925                     "value": null
4926                   },
4927                   {
4928                     "color": "red",
4929                     "value": 80
4930                   }
4931                 ]
4932               },
4933               "unit": "short"
4934             },
4935             "overrides": []
4936           },
4937           "gridPos": {
4938             "h": 8,
4939             "w": 12,
4940             "x": 0,
4941             "y": 40
4942           },
4943           "id": 104,
4944           "options": {
4945             "legend": {
4946               "calcs": [
4947                 "mean",
4948                 "lastNotNull",
4949                 "max"
4950               ],
4951               "displayMode": "table",
4952               "placement": "bottom"
4953             },
4954             "tooltip": {
4955               "mode": "single"
4956             }
4957           },
4958           "pluginVersion": "8.1.3",
4959           "targets": [
4960             {
4961               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4962               "interval": "",
4963               "legendFormat": "{{listener}}",
4964               "refId": "A"
4965             }
4966           ],
4967           "timeFrom": null,
4968           "timeShift": null,
4969           "title": "Connections creation rate per listener",
4970           "type": "timeseries"
4971         },
4972         {
4973           "datasource": "Prometheus",
4974           "fieldConfig": {
4975             "defaults": {
4976               "color": {
4977                 "mode": "palette-classic"
4978               },
4979               "custom": {
4980                 "axisLabel": "",
4981                 "axisPlacement": "auto",
4982                 "barAlignment": 0,
4983                 "drawStyle": "line",
4984                 "fillOpacity": 10,
4985                 "gradientMode": "none",
4986                 "hideFrom": {
4987                   "legend": false,
4988                   "tooltip": false,
4989                   "viz": false
4990                 },
4991                 "lineInterpolation": "linear",
4992                 "lineWidth": 1,
4993                 "pointSize": 5,
4994                 "scaleDistribution": {
4995                   "type": "linear"
4996                 },
4997                 "showPoints": "never",
4998                 "spanNulls": true,
4999                 "stacking": {
5000                   "group": "A",
5001                   "mode": "none"
5002                 },
5003                 "thresholdsStyle": {
5004                   "mode": "off"
5005                 }
5006               },
5007               "links": [],
5008               "mappings": [],
5009               "thresholds": {
5010                 "mode": "absolute",
5011                 "steps": [
5012                   {
5013                     "color": "green",
5014                     "value": null
5015                   },
5016                   {
5017                     "color": "red",
5018                     "value": 80
5019                   }
5020                 ]
5021               },
5022               "unit": "short"
5023             },
5024             "overrides": []
5025           },
5026           "gridPos": {
5027             "h": 8,
5028             "w": 12,
5029             "x": 12,
5030             "y": 40
5031           },
5032           "id": 106,
5033           "options": {
5034             "legend": {
5035               "calcs": [
5036                 "mean",
5037                 "lastNotNull",
5038                 "max"
5039               ],
5040               "displayMode": "table",
5041               "placement": "bottom"
5042             },
5043             "tooltip": {
5044               "mode": "single"
5045             }
5046           },
5047           "pluginVersion": "8.1.3",
5048           "targets": [
5049             {
5050               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5051               "interval": "",
5052               "legendFormat": "{{instance}}",
5053               "refId": "A"
5054             }
5055           ],
5056           "timeFrom": null,
5057           "timeShift": null,
5058           "title": "Connections creation rate per instance",
5059           "type": "timeseries"
5060         },
5061         {
5062           "datasource": "Prometheus",
5063           "fieldConfig": {
5064             "defaults": {
5065               "color": {
5066                 "mode": "palette-classic"
5067               },
5068               "custom": {
5069                 "axisLabel": "",
5070                 "axisPlacement": "auto",
5071                 "barAlignment": 0,
5072                 "drawStyle": "line",
5073                 "fillOpacity": 10,
5074                 "gradientMode": "none",
5075                 "hideFrom": {
5076                   "legend": false,
5077                   "tooltip": false,
5078                   "viz": false
5079                 },
5080                 "lineInterpolation": "linear",
5081                 "lineWidth": 1,
5082                 "pointSize": 5,
5083                 "scaleDistribution": {
5084                   "type": "linear"
5085                 },
5086                 "showPoints": "never",
5087                 "spanNulls": true,
5088                 "stacking": {
5089                   "group": "A",
5090                   "mode": "none"
5091                 },
5092                 "thresholdsStyle": {
5093                   "mode": "off"
5094                 }
5095               },
5096               "links": [],
5097               "mappings": [],
5098               "thresholds": {
5099                 "mode": "absolute",
5100                 "steps": [
5101                   {
5102                     "color": "green",
5103                     "value": null
5104                   },
5105                   {
5106                     "color": "red",
5107                     "value": 80
5108                   }
5109                 ]
5110               },
5111               "unit": "short"
5112             },
5113             "overrides": []
5114           },
5115           "gridPos": {
5116             "h": 8,
5117             "w": 12,
5118             "x": 0,
5119             "y": 48
5120           },
5121           "id": 108,
5122           "options": {
5123             "legend": {
5124               "calcs": [
5125                 "mean",
5126                 "lastNotNull",
5127                 "max"
5128               ],
5129               "displayMode": "table",
5130               "placement": "bottom"
5131             },
5132             "tooltip": {
5133               "mode": "single"
5134             }
5135           },
5136           "pluginVersion": "8.1.3",
5137           "targets": [
5138             {
5139               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5140               "interval": "",
5141               "legendFormat": "{{listener}}",
5142               "refId": "A"
5143             }
5144           ],
5145           "timeFrom": null,
5146           "timeShift": null,
5147           "title": "Connections close rate per listener",
5148           "type": "timeseries"
5149         },
5150         {
5151           "datasource": "Prometheus",
5152           "fieldConfig": {
5153             "defaults": {
5154               "color": {
5155                 "mode": "palette-classic"
5156               },
5157               "custom": {
5158                 "axisLabel": "",
5159                 "axisPlacement": "auto",
5160                 "barAlignment": 0,
5161                 "drawStyle": "line",
5162                 "fillOpacity": 10,
5163                 "gradientMode": "none",
5164                 "hideFrom": {
5165                   "legend": false,
5166                   "tooltip": false,
5167                   "viz": false
5168                 },
5169                 "lineInterpolation": "linear",
5170                 "lineWidth": 1,
5171                 "pointSize": 5,
5172                 "scaleDistribution": {
5173                   "type": "linear"
5174                 },
5175                 "showPoints": "never",
5176                 "spanNulls": true,
5177                 "stacking": {
5178                   "group": "A",
5179                   "mode": "none"
5180                 },
5181                 "thresholdsStyle": {
5182                   "mode": "off"
5183                 }
5184               },
5185               "links": [],
5186               "mappings": [],
5187               "thresholds": {
5188                 "mode": "absolute",
5189                 "steps": [
5190                   {
5191                     "color": "green",
5192                     "value": null
5193                   },
5194                   {
5195                     "color": "red",
5196                     "value": 80
5197                   }
5198                 ]
5199               },
5200               "unit": "short"
5201             },
5202             "overrides": []
5203           },
5204           "gridPos": {
5205             "h": 8,
5206             "w": 12,
5207             "x": 12,
5208             "y": 48
5209           },
5210           "id": 110,
5211           "options": {
5212             "legend": {
5213               "calcs": [
5214                 "mean",
5215                 "lastNotNull",
5216                 "max"
5217               ],
5218               "displayMode": "table",
5219               "placement": "bottom"
5220             },
5221             "tooltip": {
5222               "mode": "single"
5223             }
5224           },
5225           "pluginVersion": "8.1.3",
5226           "targets": [
5227             {
5228               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5229               "interval": "",
5230               "legendFormat": "{{instance}}",
5231               "refId": "A"
5232             }
5233           ],
5234           "timeFrom": null,
5235           "timeShift": null,
5236           "title": "Connections close rate per instance",
5237           "type": "timeseries"
5238         },
5239         {
5240           "datasource": "Prometheus",
5241           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5242           "fieldConfig": {
5243             "defaults": {
5244               "color": {
5245                 "mode": "palette-classic"
5246               },
5247               "custom": {
5248                 "axisLabel": "",
5249                 "axisPlacement": "auto",
5250                 "barAlignment": 0,
5251                 "drawStyle": "line",
5252                 "fillOpacity": 10,
5253                 "gradientMode": "none",
5254                 "hideFrom": {
5255                   "legend": false,
5256                   "tooltip": false,
5257                   "viz": false
5258                 },
5259                 "lineInterpolation": "linear",
5260                 "lineWidth": 1,
5261                 "pointSize": 5,
5262                 "scaleDistribution": {
5263                   "type": "linear"
5264                 },
5265                 "showPoints": "never",
5266                 "spanNulls": true,
5267                 "stacking": {
5268                   "group": "A",
5269                   "mode": "none"
5270                 },
5271                 "thresholdsStyle": {
5272                   "mode": "off"
5273                 }
5274               },
5275               "links": [],
5276               "mappings": [],
5277               "min": 0,
5278               "thresholds": {
5279                 "mode": "absolute",
5280                 "steps": [
5281                   {
5282                     "color": "green",
5283                     "value": null
5284                   },
5285                   {
5286                     "color": "red",
5287                     "value": 80
5288                   }
5289                 ]
5290               },
5291               "unit": "percent"
5292             },
5293             "overrides": []
5294           },
5295           "gridPos": {
5296             "h": 8,
5297             "w": 12,
5298             "x": 0,
5299             "y": 56
5300           },
5301           "id": 112,
5302           "options": {
5303             "legend": {
5304               "calcs": [
5305                 "mean",
5306                 "lastNotNull",
5307                 "max"
5308               ],
5309               "displayMode": "table",
5310               "placement": "bottom"
5311             },
5312             "tooltip": {
5313               "mode": "single"
5314             }
5315           },
5316           "pluginVersion": "8.1.3",
5317           "targets": [
5318             {
5319               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5320               "interval": "",
5321               "legendFormat": "{{instance}} - {{listener}}",
5322               "refId": "A"
5323             }
5324           ],
5325           "timeFrom": null,
5326           "timeShift": null,
5327           "title": "Acceptor Blocked Percentage",
5328           "type": "timeseries"
5329         },
5330         {
5331           "datasource": "Prometheus",
5332           "fieldConfig": {
5333             "defaults": {
5334               "color": {
5335                 "mode": "palette-classic"
5336               },
5337               "custom": {
5338                 "axisLabel": "",
5339                 "axisPlacement": "auto",
5340                 "barAlignment": 0,
5341                 "drawStyle": "line",
5342                 "fillOpacity": 10,
5343                 "gradientMode": "none",
5344                 "hideFrom": {
5345                   "legend": false,
5346                   "tooltip": false,
5347                   "viz": false
5348                 },
5349                 "lineInterpolation": "linear",
5350                 "lineWidth": 1,
5351                 "pointSize": 5,
5352                 "scaleDistribution": {
5353                   "type": "linear"
5354                 },
5355                 "showPoints": "never",
5356                 "spanNulls": true,
5357                 "stacking": {
5358                   "group": "A",
5359                   "mode": "none"
5360                 },
5361                 "thresholdsStyle": {
5362                   "mode": "off"
5363                 }
5364               },
5365               "links": [],
5366               "mappings": [],
5367               "thresholds": {
5368                 "mode": "absolute",
5369                 "steps": [
5370                   {
5371                     "color": "green",
5372                     "value": null
5373                   },
5374                   {
5375                     "color": "red",
5376                     "value": 80
5377                   }
5378                 ]
5379               },
5380               "unit": "short"
5381             },
5382             "overrides": []
5383           },
5384           "gridPos": {
5385             "h": 8,
5386             "w": 12,
5387             "x": 12,
5388             "y": 56
5389           },
5390           "id": 114,
5391           "options": {
5392             "legend": {
5393               "calcs": [
5394                 "mean",
5395                 "lastNotNull",
5396                 "max"
5397               ],
5398               "displayMode": "table",
5399               "placement": "bottom"
5400             },
5401             "tooltip": {
5402               "mode": "single"
5403             }
5404           },
5405           "pluginVersion": "8.1.3",
5406           "targets": [
5407             {
5408               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5409               "interval": "",
5410               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5411               "refId": "A"
5412             }
5413           ],
5414           "timeFrom": null,
5415           "timeShift": null,
5416           "title": "Connections per client version",
5417           "type": "timeseries"
5418         }
5419       ],
5420       "title": "Connections",
5421       "type": "row"
5422     },
5423     {
5424       "collapsed": true,
5425       "datasource": null,
5426       "gridPos": {
5427         "h": 1,
5428         "w": 24,
5429         "x": 0,
5430         "y": 36
5431       },
5432       "id": 120,
5433       "panels": [
5434         {
5435           "datasource": "Prometheus",
5436           "description": "Number of consumer groups per group coordinator",
5437           "fieldConfig": {
5438             "defaults": {
5439               "color": {
5440                 "mode": "palette-classic"
5441               },
5442               "custom": {
5443                 "axisLabel": "",
5444                 "axisPlacement": "auto",
5445                 "barAlignment": 0,
5446                 "drawStyle": "line",
5447                 "fillOpacity": 10,
5448                 "gradientMode": "none",
5449                 "hideFrom": {
5450                   "legend": false,
5451                   "tooltip": false,
5452                   "viz": false
5453                 },
5454                 "lineInterpolation": "linear",
5455                 "lineWidth": 1,
5456                 "pointSize": 5,
5457                 "scaleDistribution": {
5458                   "type": "linear"
5459                 },
5460                 "showPoints": "never",
5461                 "spanNulls": true,
5462                 "stacking": {
5463                   "group": "A",
5464                   "mode": "none"
5465                 },
5466                 "thresholdsStyle": {
5467                   "mode": "off"
5468                 }
5469               },
5470               "links": [],
5471               "mappings": [],
5472               "thresholds": {
5473                 "mode": "absolute",
5474                 "steps": [
5475                   {
5476                     "color": "green",
5477                     "value": null
5478                   },
5479                   {
5480                     "color": "red",
5481                     "value": 80
5482                   }
5483                 ]
5484               },
5485               "unit": "short"
5486             },
5487             "overrides": []
5488           },
5489           "gridPos": {
5490             "h": 8,
5491             "w": 12,
5492             "x": 0,
5493             "y": 33
5494           },
5495           "id": 116,
5496           "options": {
5497             "legend": {
5498               "calcs": [
5499                 "mean",
5500                 "lastNotNull",
5501                 "max"
5502               ],
5503               "displayMode": "table",
5504               "placement": "bottom"
5505             },
5506             "tooltip": {
5507               "mode": "single"
5508             }
5509           },
5510           "pluginVersion": "8.1.3",
5511           "targets": [
5512             {
5513               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5514               "instant": false,
5515               "interval": "",
5516               "legendFormat": "{{instance}}",
5517               "refId": "A"
5518             }
5519           ],
5520           "timeFrom": null,
5521           "timeShift": null,
5522           "title": "Consumer groups number per coordinator",
5523           "type": "timeseries"
5524         },
5525         {
5526           "datasource": "Prometheus",
5527           "description": "Number of consumer group per state",
5528           "fieldConfig": {
5529             "defaults": {
5530               "color": {
5531                 "mode": "palette-classic"
5532               },
5533               "custom": {
5534                 "axisLabel": "",
5535                 "axisPlacement": "auto",
5536                 "barAlignment": 0,
5537                 "drawStyle": "line",
5538                 "fillOpacity": 10,
5539                 "gradientMode": "none",
5540                 "hideFrom": {
5541                   "legend": false,
5542                   "tooltip": false,
5543                   "viz": false
5544                 },
5545                 "lineInterpolation": "linear",
5546                 "lineWidth": 1,
5547                 "pointSize": 5,
5548                 "scaleDistribution": {
5549                   "type": "linear"
5550                 },
5551                 "showPoints": "never",
5552                 "spanNulls": true,
5553                 "stacking": {
5554                   "group": "A",
5555                   "mode": "none"
5556                 },
5557                 "thresholdsStyle": {
5558                   "mode": "off"
5559                 }
5560               },
5561               "links": [],
5562               "mappings": [],
5563               "thresholds": {
5564                 "mode": "absolute",
5565                 "steps": [
5566                   {
5567                     "color": "green",
5568                     "value": null
5569                   },
5570                   {
5571                     "color": "red",
5572                     "value": 80
5573                   }
5574                 ]
5575               },
5576               "unit": "short"
5577             },
5578             "overrides": []
5579           },
5580           "gridPos": {
5581             "h": 8,
5582             "w": 12,
5583             "x": 12,
5584             "y": 33
5585           },
5586           "id": 118,
5587           "options": {
5588             "legend": {
5589               "calcs": [
5590                 "mean",
5591                 "lastNotNull",
5592                 "max"
5593               ],
5594               "displayMode": "table",
5595               "placement": "bottom"
5596             },
5597             "tooltip": {
5598               "mode": "single"
5599             }
5600           },
5601           "pluginVersion": "8.1.3",
5602           "targets": [
5603             {
5604               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5605               "instant": false,
5606               "interval": "",
5607               "legendFormat": "stable",
5608               "refId": "A"
5609             },
5610             {
5611               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5612               "interval": "",
5613               "legendFormat": "preparing-rebalance",
5614               "refId": "B"
5615             },
5616             {
5617               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5618               "interval": "",
5619               "legendFormat": "dead",
5620               "refId": "C"
5621             },
5622             {
5623               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5624               "interval": "",
5625               "legendFormat": "completing-rebalance",
5626               "refId": "D"
5627             },
5628             {
5629               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5630               "interval": "",
5631               "legendFormat": "empty",
5632               "refId": "E"
5633             }
5634           ],
5635           "timeFrom": null,
5636           "timeShift": null,
5637           "title": "Nb consumer groups per state",
5638           "type": "timeseries"
5639         }
5640       ],
5641       "title": "Group Coordinator",
5642       "type": "row"
5643     },
5644     {
5645       "collapsed": true,
5646       "datasource": null,
5647       "gridPos": {
5648         "h": 1,
5649         "w": 24,
5650         "x": 0,
5651         "y": 37
5652       },
5653       "id": 46,
5654       "panels": [
5655         {
5656           "datasource": "Prometheus",
5657           "description": "The number of messages produced converted to match the log.message.format.version.",
5658           "fieldConfig": {
5659             "defaults": {
5660               "color": {
5661                 "mode": "palette-classic"
5662               },
5663               "custom": {
5664                 "axisLabel": "",
5665                 "axisPlacement": "auto",
5666                 "barAlignment": 0,
5667                 "drawStyle": "line",
5668                 "fillOpacity": 10,
5669                 "gradientMode": "none",
5670                 "hideFrom": {
5671                   "legend": false,
5672                   "tooltip": false,
5673                   "viz": false
5674                 },
5675                 "lineInterpolation": "linear",
5676                 "lineWidth": 1,
5677                 "pointSize": 5,
5678                 "scaleDistribution": {
5679                   "type": "linear"
5680                 },
5681                 "showPoints": "never",
5682                 "spanNulls": true,
5683                 "stacking": {
5684                   "group": "A",
5685                   "mode": "none"
5686                 },
5687                 "thresholdsStyle": {
5688                   "mode": "off"
5689                 }
5690               },
5691               "links": [],
5692               "mappings": [],
5693               "min": 0,
5694               "thresholds": {
5695                 "mode": "absolute",
5696                 "steps": [
5697                   {
5698                     "color": "green",
5699                     "value": null
5700                   },
5701                   {
5702                     "color": "red",
5703                     "value": 80
5704                   }
5705                 ]
5706               },
5707               "unit": "ops"
5708             },
5709             "overrides": []
5710           },
5711           "gridPos": {
5712             "h": 8,
5713             "w": 8,
5714             "x": 0,
5715             "y": 34
5716           },
5717           "id": 48,
5718           "options": {
5719             "legend": {
5720               "calcs": [
5721                 "mean",
5722                 "lastNotNull",
5723                 "max"
5724               ],
5725               "displayMode": "list",
5726               "placement": "bottom"
5727             },
5728             "tooltip": {
5729               "mode": "single"
5730             }
5731           },
5732           "pluginVersion": "8.1.3",
5733           "targets": [
5734             {
5735               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5736               "hide": false,
5737               "interval": "",
5738               "legendFormat": "{{topic}}",
5739               "refId": "A"
5740             }
5741           ],
5742           "timeFrom": null,
5743           "timeShift": null,
5744           "title": "Number of produced message conversion",
5745           "type": "timeseries"
5746         },
5747         {
5748           "datasource": "Prometheus",
5749           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5750           "fieldConfig": {
5751             "defaults": {
5752               "color": {
5753                 "mode": "palette-classic"
5754               },
5755               "custom": {
5756                 "axisLabel": "",
5757                 "axisPlacement": "auto",
5758                 "barAlignment": 0,
5759                 "drawStyle": "line",
5760                 "fillOpacity": 10,
5761                 "gradientMode": "none",
5762                 "hideFrom": {
5763                   "legend": false,
5764                   "tooltip": false,
5765                   "viz": false
5766                 },
5767                 "lineInterpolation": "linear",
5768                 "lineWidth": 1,
5769                 "pointSize": 5,
5770                 "scaleDistribution": {
5771                   "type": "linear"
5772                 },
5773                 "showPoints": "never",
5774                 "spanNulls": true,
5775                 "stacking": {
5776                   "group": "A",
5777                   "mode": "none"
5778                 },
5779                 "thresholdsStyle": {
5780                   "mode": "off"
5781                 }
5782               },
5783               "links": [],
5784               "mappings": [],
5785               "min": 0,
5786               "thresholds": {
5787                 "mode": "absolute",
5788                 "steps": [
5789                   {
5790                     "color": "green",
5791                     "value": null
5792                   },
5793                   {
5794                     "color": "red",
5795                     "value": 80
5796                   }
5797                 ]
5798               },
5799               "unit": "ops"
5800             },
5801             "overrides": []
5802           },
5803           "gridPos": {
5804             "h": 8,
5805             "w": 8,
5806             "x": 8,
5807             "y": 34
5808           },
5809           "id": 51,
5810           "options": {
5811             "legend": {
5812               "calcs": [
5813                 "mean",
5814                 "lastNotNull",
5815                 "max"
5816               ],
5817               "displayMode": "list",
5818               "placement": "bottom"
5819             },
5820             "tooltip": {
5821               "mode": "single"
5822             }
5823           },
5824           "pluginVersion": "8.1.3",
5825           "targets": [
5826             {
5827               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5828               "hide": false,
5829               "interval": "",
5830               "legendFormat": "{{topic}}",
5831               "refId": "A"
5832             }
5833           ],
5834           "timeFrom": null,
5835           "timeShift": null,
5836           "title": "Number of consumed message conversion",
5837           "type": "timeseries"
5838         },
5839         {
5840           "cacheTimeout": null,
5841           "datasource": null,
5842           "description": "Number of connection per client version",
5843           "fieldConfig": {
5844             "defaults": {
5845               "color": {
5846                 "mode": "palette-classic"
5847               },
5848               "custom": {
5849                 "hideFrom": {
5850                   "legend": false,
5851                   "tooltip": false,
5852                   "viz": false
5853                 }
5854               },
5855               "decimals": 0,
5856               "mappings": [],
5857               "unit": "short"
5858             },
5859             "overrides": []
5860           },
5861           "gridPos": {
5862             "h": 8,
5863             "w": 8,
5864             "x": 16,
5865             "y": 34
5866           },
5867           "id": 96,
5868           "interval": null,
5869           "links": [],
5870           "options": {
5871             "legend": {
5872               "calcs": [],
5873               "displayMode": "table",
5874               "placement": "bottom",
5875               "values": [
5876                 "value"
5877               ]
5878             },
5879             "pieType": "pie",
5880             "reduceOptions": {
5881               "calcs": [
5882                 "lastNotNull"
5883               ],
5884               "fields": "",
5885               "values": false
5886             },
5887             "tooltip": {
5888               "mode": "single"
5889             }
5890           },
5891           "targets": [
5892             {
5893               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
5894               "interval": "",
5895               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
5896               "refId": "A"
5897             }
5898           ],
5899           "timeFrom": null,
5900           "timeShift": null,
5901           "title": "Client version repartition",
5902           "type": "piechart"
5903         }
5904       ],
5905       "title": "Message Conversion",
5906       "type": "row"
5907     }
5908   ],
5909   "refresh": "1m",
5910   "schemaVersion": 30,
5911   "style": "dark",
5912   "tags": [],
5913   "templating": {
5914     "list": [
5915       {
5916         "allValue": null,
5917         "current": {
5918           "selected": false,
5919           "text": "dev",
5920           "value": "dev"
5921         },
5922         "datasource": "Prometheus",
5923         "definition": "label_values(env)",
5924         "description": null,
5925         "error": null,
5926         "hide": 0,
5927         "includeAll": false,
5928         "label": "Environment",
5929         "multi": false,
5930         "name": "env",
5931         "options": [],
5932         "query": {
5933           "query": "label_values(env)",
5934           "refId": "Prometheus-env-Variable-Query"
5935         },
5936         "refresh": 1,
5937         "regex": "",
5938         "skipUrlSync": false,
5939         "sort": 0,
5940         "tagValuesQuery": "",
5941         "tagsQuery": "",
5942         "type": "query",
5943         "useTags": false
5944       },
5945       {
5946         "allValue": null,
5947         "current": {
5948           "selected": true,
5949           "text": [
5950             "All"
5951           ],
5952           "value": [
5953             "$__all"
5954           ]
5955         },
5956         "datasource": "Prometheus",
5957         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
5958         "description": null,
5959         "error": null,
5960         "hide": 0,
5961         "includeAll": true,
5962         "label": "Instance",
5963         "multi": true,
5964         "name": "instance",
5965         "options": [],
5966         "query": {
5967           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
5968           "refId": "Prometheus-instance-Variable-Query"
5969         },
5970         "refresh": 1,
5971         "regex": "",
5972         "skipUrlSync": false,
5973         "sort": 0,
5974         "tagValuesQuery": "",
5975         "tagsQuery": "",
5976         "type": "query",
5977         "useTags": false
5978       },
5979       {
5980         "allValue": "",
5981         "current": {
5982           "selected": true,
5983           "text": [
5984             "0.95"
5985           ],
5986           "value": [
5987             "0.95"
5988           ]
5989         },
5990         "datasource": "Prometheus",
5991         "definition": "label_values(quantile)",
5992         "description": null,
5993         "error": null,
5994         "hide": 0,
5995         "includeAll": true,
5996         "label": "Percentile",
5997         "multi": true,
5998         "name": "percentile",
5999         "options": [],
6000         "query": {
6001           "query": "label_values(quantile)",
6002           "refId": "Prometheus-percentile-Variable-Query"
6003         },
6004         "refresh": 1,
6005         "regex": "",
6006         "skipUrlSync": false,
6007         "sort": 0,
6008         "tagValuesQuery": "",
6009         "tagsQuery": "",
6010         "type": "query",
6011         "useTags": false
6012       }
6013     ]
6014   },
6015   "time": {
6016     "from": "now-12h",
6017     "to": "now"
6018   },
6019   "timepicker": {
6020     "refresh_intervals": [
6021       "5s",
6022       "10s",
6023       "30s",
6024       "1m",
6025       "5m",
6026       "15m",
6027       "30m",
6028       "1h",
6029       "2h",
6030       "1d"
6031     ],
6032     "time_options": [
6033       "5m",
6034       "15m",
6035       "1h",
6036       "6h",
6037       "12h",
6038       "24h",
6039       "2d",
6040       "7d",
6041       "30d"
6042     ]
6043   },
6044   "timezone": "browser",
6045   "title": "Kafka cluster",
6046   "uid": "qu-QZdfZz",
6047   "version": 2
6048 }