juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of active controllers in the cluster.",
  64       "fieldConfig": {
  65         "defaults": {
  66           "mappings": [
  67             {
  68               "options": {
  69                 "match": "null",
  70                 "result": {
  71                   "text": "N/A"
  72                 }
  73               },
  74               "type": "special"
  75             }
  76           ],
  77           "thresholds": {
  78             "mode": "absolute",
  79             "steps": [
  80               {
  81                 "color": "#299c46",
  82                 "value": null
  83               },
  84               {
  85                 "color": "#e5ac0e",
  86                 "value": 2
  87               },
  88               {
  89                 "color": "#bf1b00"
  90               }
  91             ]
  92           },
  93           "unit": "none"
  94         },
  95         "overrides": []
  96       },
  97       "gridPos": {
  98         "h": 4,
  99         "w": 4,
 100         "x": 0,
 101         "y": 1
 102       },
 103       "id": 12,
 104       "links": [],
 105       "maxDataPoints": 100,
 106       "options": {
 107         "colorMode": "value",
 108         "graphMode": "none",
 109         "justifyMode": "auto",
 110         "orientation": "vertical",
 111         "reduceOptions": {
 112           "calcs": [
 113             "lastNotNull"
 114           ],
 115           "fields": "",
 116           "values": false
 117         },
 118         "text": {},
 119         "textMode": "value_and_name"
 120       },
 121       "pluginVersion": "10.0.3",
 122       "targets": [
 123         {
 124           "datasource": {
 125             "type": "prometheus",
 126             "uid": "PBFA97CFB590B2093"
 127           },
 128           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 129           "format": "time_series",
 130           "instant": true,
 131           "interval": "",
 132           "intervalFactor": 1,
 133           "legendFormat": "{{instance}}",
 134           "refId": "A"
 135         }
 136       ],
 137       "title": "Active Controllers",
 138       "type": "stat"
 139     },
 140     {
 141       "datasource": {
 142         "type": "prometheus",
 143         "uid": "PBFA97CFB590B2093"
 144       },
 145       "description": "Number of partitions under min insync replicas.",
 146       "fieldConfig": {
 147         "defaults": {
 148           "color": {
 149             "mode": "thresholds"
 150           },
 151           "mappings": [
 152             {
 153               "options": {
 154                 "match": "null",
 155                 "result": {
 156                   "text": "N/A"
 157                 }
 158               },
 159               "type": "special"
 160             }
 161           ],
 162           "thresholds": {
 163             "mode": "absolute",
 164             "steps": [
 165               {
 166                 "color": "#508642",
 167                 "value": null
 168               },
 169               {
 170                 "color": "rgba(237, 129, 40, 0.89)",
 171                 "value": 1
 172               },
 173               {
 174                 "color": "#bf1b00",
 175                 "value": 5
 176               }
 177             ]
 178           },
 179           "unit": "none"
 180         },
 181         "overrides": []
 182       },
 183       "gridPos": {
 184         "h": 4,
 185         "w": 4,
 186         "x": 4,
 187         "y": 1
 188       },
 189       "id": 32,
 190       "links": [],
 191       "maxDataPoints": 100,
 192       "options": {
 193         "colorMode": "value",
 194         "graphMode": "none",
 195         "justifyMode": "auto",
 196         "orientation": "horizontal",
 197         "reduceOptions": {
 198           "calcs": [
 199             "lastNotNull"
 200           ],
 201           "fields": "",
 202           "values": false
 203         },
 204         "text": {},
 205         "textMode": "auto"
 206       },
 207       "pluginVersion": "10.0.3",
 208       "targets": [
 209         {
 210           "datasource": {
 211             "type": "prometheus",
 212             "uid": "PBFA97CFB590B2093"
 213           },
 214           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 215           "format": "time_series",
 216           "hide": false,
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 2,
 220           "legendFormat": "",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Under Min ISR Partitions",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#508642",
 254                 "value": null
 255               },
 256               {
 257                 "color": "#ef843c",
 258                 "value": 1
 259               },
 260               {
 261                 "color": "#bf1b00",
 262                 "value": 1
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 4,
 273         "x": 8,
 274         "y": 1
 275       },
 276       "id": 22,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Offline Partitions Count",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 12,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#508642",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 1
 430               },
 431               {
 432                 "color": "#bf1b00",
 433                 "value": 5
 434               }
 435             ]
 436           },
 437           "unit": "none"
 438         },
 439         "overrides": []
 440       },
 441       "gridPos": {
 442         "h": 4,
 443         "w": 4,
 444         "x": 0,
 445         "y": 5
 446       },
 447       "id": 20,
 448       "links": [],
 449       "maxDataPoints": 100,
 450       "options": {
 451         "colorMode": "value",
 452         "graphMode": "none",
 453         "justifyMode": "auto",
 454         "orientation": "horizontal",
 455         "reduceOptions": {
 456           "calcs": [
 457             "lastNotNull"
 458           ],
 459           "fields": "",
 460           "values": false
 461         },
 462         "text": {},
 463         "textMode": "auto"
 464       },
 465       "pluginVersion": "10.0.3",
 466       "targets": [
 467         {
 468           "datasource": {
 469             "type": "prometheus",
 470             "uid": "PBFA97CFB590B2093"
 471           },
 472           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 473           "format": "time_series",
 474           "hide": false,
 475           "instant": true,
 476           "interval": "",
 477           "intervalFactor": 2,
 478           "legendFormat": "",
 479           "refId": "A"
 480         }
 481       ],
 482       "title": "Under Replicated Partitions",
 483       "type": "stat"
 484     },
 485     {
 486       "datasource": {
 487         "type": "prometheus",
 488         "uid": "PBFA97CFB590B2093"
 489       },
 490       "description": "Number of Brokers Online",
 491       "fieldConfig": {
 492         "defaults": {
 493           "color": {
 494             "mode": "thresholds"
 495           },
 496           "mappings": [
 497             {
 498               "options": {
 499                 "match": "null",
 500                 "result": {
 501                   "text": "N/A"
 502                 }
 503               },
 504               "type": "special"
 505             }
 506           ],
 507           "thresholds": {
 508             "mode": "absolute",
 509             "steps": [
 510               {
 511                 "color": "#d44a3a",
 512                 "value": null
 513               },
 514               {
 515                 "color": "rgba(237, 129, 40, 0.89)",
 516                 "value": 0
 517               },
 518               {
 519                 "color": "#299c46",
 520                 "value": 2
 521               }
 522             ]
 523           },
 524           "unit": "none"
 525         },
 526         "overrides": []
 527       },
 528       "gridPos": {
 529         "h": 4,
 530         "w": 4,
 531         "x": 4,
 532         "y": 5
 533       },
 534       "id": 14,
 535       "links": [],
 536       "maxDataPoints": 100,
 537       "options": {
 538         "colorMode": "value",
 539         "graphMode": "none",
 540         "justifyMode": "auto",
 541         "orientation": "horizontal",
 542         "reduceOptions": {
 543           "calcs": [
 544             "lastNotNull"
 545           ],
 546           "fields": "",
 547           "values": false
 548         },
 549         "text": {},
 550         "textMode": "auto"
 551       },
 552       "pluginVersion": "10.0.3",
 553       "repeatDirection": "h",
 554       "targets": [
 555         {
 556           "datasource": {
 557             "type": "prometheus",
 558             "uid": "PBFA97CFB590B2093"
 559           },
 560           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 561           "format": "time_series",
 562           "instant": true,
 563           "interval": "",
 564           "intervalFactor": 1,
 565           "legendFormat": "",
 566           "refId": "A"
 567         }
 568       ],
 569       "title": "Brokers Online",
 570       "type": "stat"
 571     },
 572     {
 573       "datasource": {
 574         "type": "prometheus",
 575         "uid": "PBFA97CFB590B2093"
 576       },
 577       "description": "Partitions that are online",
 578       "fieldConfig": {
 579         "defaults": {
 580           "color": {
 581             "mode": "thresholds"
 582           },
 583           "mappings": [
 584             {
 585               "options": {
 586                 "match": "null",
 587                 "result": {
 588                   "text": "N/A"
 589                 }
 590               },
 591               "type": "special"
 592             }
 593           ],
 594           "thresholds": {
 595             "mode": "absolute",
 596             "steps": [
 597               {
 598                 "color": "#d44a3a",
 599                 "value": null
 600               },
 601               {
 602                 "color": "rgba(237, 129, 40, 0.89)",
 603                 "value": 0
 604               },
 605               {
 606                 "color": "#299c46",
 607                 "value": 0
 608               }
 609             ]
 610           },
 611           "unit": "none"
 612         },
 613         "overrides": []
 614       },
 615       "gridPos": {
 616         "h": 4,
 617         "w": 4,
 618         "x": 8,
 619         "y": 5
 620       },
 621       "id": 18,
 622       "links": [],
 623       "maxDataPoints": 100,
 624       "options": {
 625         "colorMode": "value",
 626         "graphMode": "none",
 627         "justifyMode": "auto",
 628         "orientation": "horizontal",
 629         "reduceOptions": {
 630           "calcs": [
 631             "lastNotNull"
 632           ],
 633           "fields": "",
 634           "values": false
 635         },
 636         "text": {},
 637         "textMode": "auto"
 638       },
 639       "pluginVersion": "10.0.3",
 640       "targets": [
 641         {
 642           "datasource": {
 643             "type": "prometheus",
 644             "uid": "PBFA97CFB590B2093"
 645           },
 646           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 647           "format": "time_series",
 648           "instant": true,
 649           "interval": "",
 650           "intervalFactor": 1,
 651           "legendFormat": "",
 652           "refId": "A"
 653         }
 654       ],
 655       "title": "Online Partitions",
 656       "type": "stat"
 657     },
 658     {
 659       "datasource": {
 660         "type": "prometheus",
 661         "uid": "PBFA97CFB590B2093"
 662       },
 663       "description": "Unclean leader election rate",
 664       "fieldConfig": {
 665         "defaults": {
 666           "color": {
 667             "mode": "thresholds"
 668           },
 669           "mappings": [
 670             {
 671               "options": {
 672                 "match": "null",
 673                 "result": {
 674                   "text": "N/A"
 675                 }
 676               },
 677               "type": "special"
 678             }
 679           ],
 680           "thresholds": {
 681             "mode": "absolute",
 682             "steps": [
 683               {
 684                 "color": "#299c46",
 685                 "value": null
 686               },
 687               {
 688                 "color": "rgba(237, 129, 40, 0.89)",
 689                 "value": 2
 690               },
 691               {
 692                 "color": "#d44a3a"
 693               }
 694             ]
 695           },
 696           "unit": "none"
 697         },
 698         "overrides": []
 699       },
 700       "gridPos": {
 701         "h": 4,
 702         "w": 4,
 703         "x": 12,
 704         "y": 5
 705       },
 706       "id": 16,
 707       "links": [],
 708       "maxDataPoints": 100,
 709       "options": {
 710         "colorMode": "value",
 711         "graphMode": "none",
 712         "justifyMode": "auto",
 713         "orientation": "horizontal",
 714         "reduceOptions": {
 715           "calcs": [
 716             "lastNotNull"
 717           ],
 718           "fields": "",
 719           "values": false
 720         },
 721         "text": {},
 722         "textMode": "auto"
 723       },
 724       "pluginVersion": "10.0.3",
 725       "targets": [
 726         {
 727           "datasource": {
 728             "type": "prometheus",
 729             "uid": "PBFA97CFB590B2093"
 730           },
 731           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 732           "format": "time_series",
 733           "instant": true,
 734           "interval": "",
 735           "intervalFactor": 1,
 736           "legendFormat": "",
 737           "refId": "A"
 738         }
 739       ],
 740       "title": "Unclean Leader Election Rate",
 741       "type": "stat"
 742     },
 743     {
 744       "datasource": {
 745         "type": "prometheus",
 746         "uid": "PBFA97CFB590B2093"
 747       },
 748       "fieldConfig": {
 749         "defaults": {
 750           "color": {
 751             "mode": "palette-classic"
 752           },
 753           "custom": {
 754             "axisCenteredZero": false,
 755             "axisColorMode": "text",
 756             "axisLabel": "Bytes/s",
 757             "axisPlacement": "auto",
 758             "barAlignment": 0,
 759             "drawStyle": "line",
 760             "fillOpacity": 10,
 761             "gradientMode": "none",
 762             "hideFrom": {
 763               "legend": false,
 764               "tooltip": false,
 765               "viz": false
 766             },
 767             "lineInterpolation": "linear",
 768             "lineWidth": 2,
 769             "pointSize": 5,
 770             "scaleDistribution": {
 771               "type": "linear"
 772             },
 773             "showPoints": "never",
 774             "spanNulls": false,
 775             "stacking": {
 776               "group": "A",
 777               "mode": "none"
 778             },
 779             "thresholdsStyle": {
 780               "mode": "off"
 781             }
 782           },
 783           "links": [],
 784           "mappings": [],
 785           "thresholds": {
 786             "mode": "absolute",
 787             "steps": [
 788               {
 789                 "color": "green",
 790                 "value": null
 791               },
 792               {
 793                 "color": "red",
 794                 "value": 80
 795               }
 796             ]
 797           },
 798           "unit": "Bps"
 799         },
 800         "overrides": []
 801       },
 802       "gridPos": {
 803         "h": 8,
 804         "w": 8,
 805         "x": 0,
 806         "y": 9
 807       },
 808       "id": 84,
 809       "links": [],
 810       "options": {
 811         "legend": {
 812           "calcs": [
 813             "mean",
 814             "lastNotNull",
 815             "max"
 816           ],
 817           "displayMode": "list",
 818           "placement": "bottom",
 819           "showLegend": true
 820         },
 821         "tooltip": {
 822           "mode": "single",
 823           "sort": "none"
 824         }
 825       },
 826       "pluginVersion": "8.1.3",
 827       "targets": [
 828         {
 829           "datasource": {
 830             "type": "prometheus",
 831             "uid": "PBFA97CFB590B2093"
 832           },
 833           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 834           "format": "time_series",
 835           "interval": "",
 836           "intervalFactor": 2,
 837           "legendFormat": "Bytes in",
 838           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 839           "refId": "A",
 840           "step": 4
 841         },
 842         {
 843           "datasource": {
 844             "type": "prometheus",
 845             "uid": "PBFA97CFB590B2093"
 846           },
 847           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 848           "format": "time_series",
 849           "hide": false,
 850           "interval": "",
 851           "intervalFactor": 2,
 852           "legendFormat": "Bytes out",
 853           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 854           "refId": "B",
 855           "step": 4
 856         }
 857       ],
 858       "title": "Broker network throughput",
 859       "type": "timeseries"
 860     },
 861     {
 862       "datasource": {
 863         "type": "prometheus",
 864         "uid": "PBFA97CFB590B2093"
 865       },
 866       "description": "",
 867       "fieldConfig": {
 868         "defaults": {
 869           "color": {
 870             "mode": "palette-classic"
 871           },
 872           "custom": {
 873             "axisCenteredZero": false,
 874             "axisColorMode": "text",
 875             "axisLabel": "",
 876             "axisPlacement": "auto",
 877             "barAlignment": 0,
 878             "drawStyle": "line",
 879             "fillOpacity": 10,
 880             "gradientMode": "none",
 881             "hideFrom": {
 882               "legend": false,
 883               "tooltip": false,
 884               "viz": false
 885             },
 886             "lineInterpolation": "linear",
 887             "lineWidth": 1,
 888             "pointSize": 5,
 889             "scaleDistribution": {
 890               "type": "linear"
 891             },
 892             "showPoints": "never",
 893             "spanNulls": true,
 894             "stacking": {
 895               "group": "A",
 896               "mode": "none"
 897             },
 898             "thresholdsStyle": {
 899               "mode": "off"
 900             }
 901           },
 902           "decimals": 0,
 903           "links": [],
 904           "mappings": [],
 905           "min": 0,
 906           "thresholds": {
 907             "mode": "absolute",
 908             "steps": [
 909               {
 910                 "color": "green"
 911               },
 912               {
 913                 "color": "red",
 914                 "value": 80
 915               }
 916             ]
 917           },
 918           "unit": "partitions"
 919         },
 920         "overrides": [
 921           {
 922             "__systemRef": "hideSeriesFrom",
 923             "matcher": {
 924               "id": "byNames",
 925               "options": {
 926                 "mode": "exclude",
 927                 "names": [
 928                   "kafka-3:80"
 929                 ],
 930                 "prefix": "All except:",
 931                 "readOnly": true
 932               }
 933             },
 934             "properties": [
 935               {
 936                 "id": "custom.hideFrom",
 937                 "value": {
 938                   "legend": false,
 939                   "tooltip": false,
 940                   "viz": true
 941                 }
 942               }
 943             ]
 944           }
 945         ]
 946       },
 947       "gridPos": {
 948         "h": 8,
 949         "w": 16,
 950         "x": 8,
 951         "y": 9
 952       },
 953       "id": 123,
 954       "options": {
 955         "legend": {
 956           "calcs": [
 957             "lastNotNull",
 958             "max"
 959           ],
 960           "displayMode": "table",
 961           "placement": "bottom",
 962           "showLegend": true
 963         },
 964         "tooltip": {
 965           "mode": "single",
 966           "sort": "none"
 967         }
 968       },
 969       "pluginVersion": "8.1.3",
 970       "targets": [
 971         {
 972           "datasource": {
 973             "type": "prometheus",
 974             "uid": "PBFA97CFB590B2093"
 975           },
 976           "editorMode": "code",
 977           "exemplar": false,
 978           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 979           "hide": false,
 980           "interval": "",
 981           "legendFormat": "{{instance}}",
 982           "range": true,
 983           "refId": "A"
 984         }
 985       ],
 986       "title": "Underreplicated Partitions",
 987       "type": "timeseries"
 988     },
 989     {
 990       "collapsed": false,
 991       "datasource": {
 992         "type": "datasource",
 993         "uid": "grafana"
 994       },
 995       "gridPos": {
 996         "h": 1,
 997         "w": 24,
 998         "x": 0,
 999         "y": 17
1000       },
1001       "id": 31,
1002       "panels": [],
1003       "targets": [
1004         {
1005           "datasource": {
1006             "type": "datasource",
1007             "uid": "grafana"
1008           },
1009           "refId": "A"
1010         }
1011       ],
1012       "title": "Request rate",
1013       "type": "row"
1014     },
1015     {
1016       "datasource": {
1017         "type": "prometheus",
1018         "uid": "PBFA97CFB590B2093"
1019       },
1020       "description": "Produce request rate.",
1021       "fieldConfig": {
1022         "defaults": {
1023           "mappings": [
1024             {
1025               "options": {
1026                 "match": "null",
1027                 "result": {
1028                   "text": "N/A"
1029                 }
1030               },
1031               "type": "special"
1032             }
1033           ],
1034           "thresholds": {
1035             "mode": "absolute",
1036             "steps": [
1037               {
1038                 "color": "green",
1039                 "value": null
1040               }
1041             ]
1042           },
1043           "unit": "none"
1044         },
1045         "overrides": []
1046       },
1047       "gridPos": {
1048         "h": 4,
1049         "w": 4,
1050         "x": 0,
1051         "y": 18
1052       },
1053       "id": 93,
1054       "links": [],
1055       "maxDataPoints": 100,
1056       "options": {
1057         "colorMode": "value",
1058         "graphMode": "area",
1059         "justifyMode": "auto",
1060         "orientation": "horizontal",
1061         "reduceOptions": {
1062           "calcs": [
1063             "lastNotNull"
1064           ],
1065           "fields": "",
1066           "values": false
1067         },
1068         "text": {},
1069         "textMode": "auto"
1070       },
1071       "pluginVersion": "10.0.3",
1072       "targets": [
1073         {
1074           "datasource": {
1075             "type": "prometheus",
1076             "uid": "PBFA97CFB590B2093"
1077           },
1078           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1079           "interval": "",
1080           "legendFormat": "",
1081           "refId": "A"
1082         }
1083       ],
1084       "title": "All Request Per Sec",
1085       "type": "stat"
1086     },
1087     {
1088       "datasource": {
1089         "type": "prometheus",
1090         "uid": "PBFA97CFB590B2093"
1091       },
1092       "description": "Produce request rate.",
1093       "fieldConfig": {
1094         "defaults": {
1095           "mappings": [
1096             {
1097               "options": {
1098                 "match": "null",
1099                 "result": {
1100                   "text": "N/A"
1101                 }
1102               },
1103               "type": "special"
1104             }
1105           ],
1106           "thresholds": {
1107             "mode": "absolute",
1108             "steps": [
1109               {
1110                 "color": "green",
1111                 "value": null
1112               }
1113             ]
1114           },
1115           "unit": "none"
1116         },
1117         "overrides": []
1118       },
1119       "gridPos": {
1120         "h": 4,
1121         "w": 4,
1122         "x": 4,
1123         "y": 18
1124       },
1125       "id": 35,
1126       "links": [],
1127       "maxDataPoints": 100,
1128       "options": {
1129         "colorMode": "value",
1130         "graphMode": "area",
1131         "justifyMode": "auto",
1132         "orientation": "horizontal",
1133         "reduceOptions": {
1134           "calcs": [
1135             "lastNotNull"
1136           ],
1137           "fields": "",
1138           "values": false
1139         },
1140         "text": {},
1141         "textMode": "auto"
1142       },
1143       "pluginVersion": "10.0.3",
1144       "targets": [
1145         {
1146           "datasource": {
1147             "type": "prometheus",
1148             "uid": "PBFA97CFB590B2093"
1149           },
1150           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1151           "interval": "",
1152           "legendFormat": "",
1153           "refId": "A"
1154         }
1155       ],
1156       "title": "Produce Request Per Sec",
1157       "type": "stat"
1158     },
1159     {
1160       "datasource": {
1161         "type": "prometheus",
1162         "uid": "PBFA97CFB590B2093"
1163       },
1164       "description": "Fetch request rate.",
1165       "fieldConfig": {
1166         "defaults": {
1167           "mappings": [
1168             {
1169               "options": {
1170                 "match": "null",
1171                 "result": {
1172                   "text": "N/A"
1173                 }
1174               },
1175               "type": "special"
1176             }
1177           ],
1178           "thresholds": {
1179             "mode": "absolute",
1180             "steps": [
1181               {
1182                 "color": "green",
1183                 "value": null
1184               }
1185             ]
1186           },
1187           "unit": "none"
1188         },
1189         "overrides": []
1190       },
1191       "gridPos": {
1192         "h": 4,
1193         "w": 4,
1194         "x": 8,
1195         "y": 18
1196       },
1197       "id": 37,
1198       "links": [],
1199       "maxDataPoints": 100,
1200       "options": {
1201         "colorMode": "value",
1202         "graphMode": "area",
1203         "justifyMode": "auto",
1204         "orientation": "horizontal",
1205         "reduceOptions": {
1206           "calcs": [
1207             "lastNotNull"
1208           ],
1209           "fields": "",
1210           "values": false
1211         },
1212         "text": {},
1213         "textMode": "auto"
1214       },
1215       "pluginVersion": "10.0.3",
1216       "targets": [
1217         {
1218           "datasource": {
1219             "type": "prometheus",
1220             "uid": "PBFA97CFB590B2093"
1221           },
1222           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1223           "interval": "",
1224           "legendFormat": "",
1225           "refId": "A"
1226         }
1227       ],
1228       "title": "Consumer Fetch Request Per Sec",
1229       "type": "stat"
1230     },
1231     {
1232       "datasource": {
1233         "type": "prometheus",
1234         "uid": "PBFA97CFB590B2093"
1235       },
1236       "description": "Fetch request rate.",
1237       "fieldConfig": {
1238         "defaults": {
1239           "mappings": [
1240             {
1241               "options": {
1242                 "match": "null",
1243                 "result": {
1244                   "text": "N/A"
1245                 }
1246               },
1247               "type": "special"
1248             }
1249           ],
1250           "thresholds": {
1251             "mode": "absolute",
1252             "steps": [
1253               {
1254                 "color": "green",
1255                 "value": null
1256               }
1257             ]
1258           },
1259           "unit": "none"
1260         },
1261         "overrides": []
1262       },
1263       "gridPos": {
1264         "h": 4,
1265         "w": 4,
1266         "x": 0,
1267         "y": 22
1268       },
1269       "id": 94,
1270       "links": [],
1271       "maxDataPoints": 100,
1272       "options": {
1273         "colorMode": "value",
1274         "graphMode": "area",
1275         "justifyMode": "auto",
1276         "orientation": "horizontal",
1277         "reduceOptions": {
1278           "calcs": [
1279             "lastNotNull"
1280           ],
1281           "fields": "",
1282           "values": false
1283         },
1284         "text": {},
1285         "textMode": "auto"
1286       },
1287       "pluginVersion": "10.0.3",
1288       "targets": [
1289         {
1290           "datasource": {
1291             "type": "prometheus",
1292             "uid": "PBFA97CFB590B2093"
1293           },
1294           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1295           "interval": "",
1296           "legendFormat": "",
1297           "refId": "A"
1298         }
1299       ],
1300       "title": "Broker Fetch Request Per Sec",
1301       "type": "stat"
1302     },
1303     {
1304       "datasource": {
1305         "type": "prometheus",
1306         "uid": "PBFA97CFB590B2093"
1307       },
1308       "description": "Offset Commit request rate.",
1309       "fieldConfig": {
1310         "defaults": {
1311           "mappings": [
1312             {
1313               "options": {
1314                 "match": "null",
1315                 "result": {
1316                   "text": "N/A"
1317                 }
1318               },
1319               "type": "special"
1320             }
1321           ],
1322           "thresholds": {
1323             "mode": "absolute",
1324             "steps": [
1325               {
1326                 "color": "green",
1327                 "value": null
1328               }
1329             ]
1330           },
1331           "unit": "none"
1332         },
1333         "overrides": []
1334       },
1335       "gridPos": {
1336         "h": 4,
1337         "w": 4,
1338         "x": 4,
1339         "y": 22
1340       },
1341       "id": 38,
1342       "links": [],
1343       "maxDataPoints": 100,
1344       "options": {
1345         "colorMode": "value",
1346         "graphMode": "area",
1347         "justifyMode": "auto",
1348         "orientation": "horizontal",
1349         "reduceOptions": {
1350           "calcs": [
1351             "lastNotNull"
1352           ],
1353           "fields": "",
1354           "values": false
1355         },
1356         "text": {},
1357         "textMode": "auto"
1358       },
1359       "pluginVersion": "10.0.3",
1360       "targets": [
1361         {
1362           "datasource": {
1363             "type": "prometheus",
1364             "uid": "PBFA97CFB590B2093"
1365           },
1366           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1367           "interval": "",
1368           "legendFormat": "",
1369           "refId": "A"
1370         }
1371       ],
1372       "title": "Offset Commit Request Per Sec",
1373       "type": "stat"
1374     },
1375     {
1376       "datasource": {
1377         "type": "prometheus",
1378         "uid": "PBFA97CFB590B2093"
1379       },
1380       "description": "Metadata request rate.",
1381       "fieldConfig": {
1382         "defaults": {
1383           "mappings": [
1384             {
1385               "options": {
1386                 "match": "null",
1387                 "result": {
1388                   "text": "N/A"
1389                 }
1390               },
1391               "type": "special"
1392             }
1393           ],
1394           "thresholds": {
1395             "mode": "absolute",
1396             "steps": [
1397               {
1398                 "color": "green",
1399                 "value": null
1400               }
1401             ]
1402           },
1403           "unit": "none"
1404         },
1405         "overrides": []
1406       },
1407       "gridPos": {
1408         "h": 4,
1409         "w": 4,
1410         "x": 8,
1411         "y": 22
1412       },
1413       "id": 36,
1414       "links": [],
1415       "maxDataPoints": 100,
1416       "options": {
1417         "colorMode": "value",
1418         "graphMode": "area",
1419         "justifyMode": "auto",
1420         "orientation": "horizontal",
1421         "reduceOptions": {
1422           "calcs": [
1423             "lastNotNull"
1424           ],
1425           "fields": "",
1426           "values": false
1427         },
1428         "text": {},
1429         "textMode": "auto"
1430       },
1431       "pluginVersion": "10.0.3",
1432       "targets": [
1433         {
1434           "datasource": {
1435             "type": "prometheus",
1436             "uid": "PBFA97CFB590B2093"
1437           },
1438           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1439           "interval": "",
1440           "legendFormat": "",
1441           "refId": "A"
1442         }
1443       ],
1444       "title": "Metadata Request Per Sec",
1445       "type": "stat"
1446     },
1447     {
1448       "collapsed": false,
1449       "datasource": {
1450         "type": "datasource",
1451         "uid": "grafana"
1452       },
1453       "gridPos": {
1454         "h": 1,
1455         "w": 24,
1456         "x": 0,
1457         "y": 26
1458       },
1459       "id": 40,
1460       "panels": [],
1461       "targets": [
1462         {
1463           "datasource": {
1464             "type": "datasource",
1465             "uid": "grafana"
1466           },
1467           "refId": "A"
1468         }
1469       ],
1470       "title": "System",
1471       "type": "row"
1472     },
1473     {
1474       "datasource": {
1475         "type": "prometheus",
1476         "uid": "PBFA97CFB590B2093"
1477       },
1478       "fieldConfig": {
1479         "defaults": {
1480           "color": {
1481             "mode": "palette-classic"
1482           },
1483           "custom": {
1484             "axisCenteredZero": false,
1485             "axisColorMode": "text",
1486             "axisLabel": "Cores",
1487             "axisPlacement": "auto",
1488             "barAlignment": 0,
1489             "drawStyle": "line",
1490             "fillOpacity": 10,
1491             "gradientMode": "none",
1492             "hideFrom": {
1493               "legend": false,
1494               "tooltip": false,
1495               "viz": false
1496             },
1497             "lineInterpolation": "linear",
1498             "lineWidth": 2,
1499             "pointSize": 5,
1500             "scaleDistribution": {
1501               "type": "linear"
1502             },
1503             "showPoints": "never",
1504             "spanNulls": false,
1505             "stacking": {
1506               "group": "A",
1507               "mode": "none"
1508             },
1509             "thresholdsStyle": {
1510               "mode": "off"
1511             }
1512           },
1513           "links": [],
1514           "mappings": [],
1515           "thresholds": {
1516             "mode": "absolute",
1517             "steps": [
1518               {
1519                 "color": "green",
1520                 "value": null
1521               },
1522               {
1523                 "color": "red",
1524                 "value": 80
1525               }
1526             ]
1527           },
1528           "unit": "percent"
1529         },
1530         "overrides": [
1531           {
1532             "matcher": {
1533               "id": "byName",
1534               "options": "localhost:7071"
1535             },
1536             "properties": [
1537               {
1538                 "id": "color",
1539                 "value": {
1540                   "fixedColor": "#629E51",
1541                   "mode": "fixed"
1542                 }
1543               }
1544             ]
1545           }
1546         ]
1547       },
1548       "gridPos": {
1549         "h": 8,
1550         "w": 8,
1551         "x": 0,
1552         "y": 27
1553       },
1554       "id": 27,
1555       "links": [],
1556       "options": {
1557         "legend": {
1558           "calcs": [
1559             "mean",
1560             "lastNotNull",
1561             "max"
1562           ],
1563           "displayMode": "table",
1564           "placement": "bottom",
1565           "showLegend": true
1566         },
1567         "tooltip": {
1568           "mode": "single",
1569           "sort": "none"
1570         }
1571       },
1572       "pluginVersion": "8.1.3",
1573       "targets": [
1574         {
1575           "datasource": {
1576             "type": "prometheus",
1577             "uid": "PBFA97CFB590B2093"
1578           },
1579           "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1580           "format": "time_series",
1581           "interval": "",
1582           "intervalFactor": 2,
1583           "legendFormat": "{{instance}}",
1584           "metric": "process_cpu_secondspersec",
1585           "refId": "A",
1586           "step": 4
1587         }
1588       ],
1589       "title": "CPU Usage",
1590       "type": "timeseries"
1591     },
1592     {
1593       "datasource": {
1594         "type": "prometheus",
1595         "uid": "PBFA97CFB590B2093"
1596       },
1597       "fieldConfig": {
1598         "defaults": {
1599           "color": {
1600             "mode": "palette-classic"
1601           },
1602           "custom": {
1603             "axisCenteredZero": false,
1604             "axisColorMode": "text",
1605             "axisLabel": "Memory",
1606             "axisPlacement": "auto",
1607             "barAlignment": 0,
1608             "drawStyle": "line",
1609             "fillOpacity": 10,
1610             "gradientMode": "none",
1611             "hideFrom": {
1612               "legend": false,
1613               "tooltip": false,
1614               "viz": false
1615             },
1616             "lineInterpolation": "linear",
1617             "lineWidth": 2,
1618             "pointSize": 5,
1619             "scaleDistribution": {
1620               "type": "linear"
1621             },
1622             "showPoints": "never",
1623             "spanNulls": false,
1624             "stacking": {
1625               "group": "A",
1626               "mode": "none"
1627             },
1628             "thresholdsStyle": {
1629               "mode": "off"
1630             }
1631           },
1632           "links": [],
1633           "mappings": [],
1634           "thresholds": {
1635             "mode": "absolute",
1636             "steps": [
1637               {
1638                 "color": "green",
1639                 "value": null
1640               },
1641               {
1642                 "color": "red",
1643                 "value": 80
1644               }
1645             ]
1646           },
1647           "unit": "bytes"
1648         },
1649         "overrides": [
1650           {
1651             "matcher": {
1652               "id": "byName",
1653               "options": "localhost:7071"
1654             },
1655             "properties": [
1656               {
1657                 "id": "color",
1658                 "value": {
1659                   "fixedColor": "#BA43A9",
1660                   "mode": "fixed"
1661                 }
1662               }
1663             ]
1664           }
1665         ]
1666       },
1667       "gridPos": {
1668         "h": 8,
1669         "w": 8,
1670         "x": 8,
1671         "y": 27
1672       },
1673       "id": 2,
1674       "links": [],
1675       "options": {
1676         "legend": {
1677           "calcs": [
1678             "mean",
1679             "lastNotNull",
1680             "max"
1681           ],
1682           "displayMode": "table",
1683           "placement": "bottom",
1684           "showLegend": true
1685         },
1686         "tooltip": {
1687           "mode": "single",
1688           "sort": "none"
1689         }
1690       },
1691       "pluginVersion": "8.1.3",
1692       "targets": [
1693         {
1694           "datasource": {
1695             "type": "prometheus",
1696             "uid": "PBFA97CFB590B2093"
1697           },
1698           "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1699           "interval": "",
1700           "intervalFactor": 2,
1701           "legendFormat": "{{instance}}",
1702           "metric": "jvm_memory_bytes_used",
1703           "refId": "A",
1704           "step": 4
1705         },
1706         {
1707           "datasource": {
1708             "type": "prometheus",
1709             "uid": "PBFA97CFB590B2093"
1710           },
1711           "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1712           "interval": "",
1713           "legendFormat": "{{instance}}",
1714           "refId": "B"
1715         }
1716       ],
1717       "title": "JVM Memory Used",
1718       "type": "timeseries"
1719     },
1720     {
1721       "datasource": {
1722         "type": "prometheus",
1723         "uid": "PBFA97CFB590B2093"
1724       },
1725       "fieldConfig": {
1726         "defaults": {
1727           "color": {
1728             "mode": "palette-classic"
1729           },
1730           "custom": {
1731             "axisCenteredZero": false,
1732             "axisColorMode": "text",
1733             "axisLabel": "% time in GC",
1734             "axisPlacement": "auto",
1735             "barAlignment": 0,
1736             "drawStyle": "line",
1737             "fillOpacity": 10,
1738             "gradientMode": "none",
1739             "hideFrom": {
1740               "legend": false,
1741               "tooltip": false,
1742               "viz": false
1743             },
1744             "lineInterpolation": "linear",
1745             "lineWidth": 2,
1746             "pointSize": 5,
1747             "scaleDistribution": {
1748               "type": "linear"
1749             },
1750             "showPoints": "never",
1751             "spanNulls": false,
1752             "stacking": {
1753               "group": "A",
1754               "mode": "none"
1755             },
1756             "thresholdsStyle": {
1757               "mode": "off"
1758             }
1759           },
1760           "links": [],
1761           "mappings": [],
1762           "thresholds": {
1763             "mode": "absolute",
1764             "steps": [
1765               {
1766                 "color": "green",
1767                 "value": null
1768               },
1769               {
1770                 "color": "red",
1771                 "value": 80
1772               }
1773             ]
1774           },
1775           "unit": "percentunit"
1776         },
1777         "overrides": [
1778           {
1779             "matcher": {
1780               "id": "byName",
1781               "options": "localhost:7071"
1782             },
1783             "properties": [
1784               {
1785                 "id": "color",
1786                 "value": {
1787                   "fixedColor": "#890F02",
1788                   "mode": "fixed"
1789                 }
1790               }
1791             ]
1792           }
1793         ]
1794       },
1795       "gridPos": {
1796         "h": 8,
1797         "w": 8,
1798         "x": 16,
1799         "y": 27
1800       },
1801       "id": 3,
1802       "links": [],
1803       "options": {
1804         "legend": {
1805           "calcs": [
1806             "mean",
1807             "lastNotNull",
1808             "max"
1809           ],
1810           "displayMode": "table",
1811           "placement": "bottom",
1812           "showLegend": true
1813         },
1814         "tooltip": {
1815           "mode": "single",
1816           "sort": "none"
1817         }
1818       },
1819       "pluginVersion": "8.1.3",
1820       "targets": [
1821         {
1822           "datasource": {
1823             "type": "prometheus",
1824             "uid": "PBFA97CFB590B2093"
1825           },
1826           "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1827           "interval": "",
1828           "intervalFactor": 2,
1829           "legendFormat": "{{instance}}",
1830           "metric": "jvm_gc_collection_seconds_sum",
1831           "refId": "A",
1832           "step": 4
1833         }
1834       ],
1835       "title": "Time spent in GC",
1836       "type": "timeseries"
1837     },
1838     {
1839       "collapsed": true,
1840       "datasource": {
1841         "type": "datasource",
1842         "uid": "grafana"
1843       },
1844       "gridPos": {
1845         "h": 1,
1846         "w": 24,
1847         "x": 0,
1848         "y": 35
1849       },
1850       "id": 29,
1851       "panels": [
1852         {
1853           "datasource": {
1854             "type": "prometheus",
1855             "uid": "PBFA97CFB590B2093"
1856           },
1857           "fieldConfig": {
1858             "defaults": {
1859               "color": {
1860                 "mode": "palette-classic"
1861               },
1862               "custom": {
1863                 "axisLabel": "Messages/s",
1864                 "axisPlacement": "auto",
1865                 "barAlignment": 0,
1866                 "drawStyle": "line",
1867                 "fillOpacity": 10,
1868                 "gradientMode": "none",
1869                 "hideFrom": {
1870                   "legend": false,
1871                   "tooltip": false,
1872                   "viz": false
1873                 },
1874                 "lineInterpolation": "linear",
1875                 "lineWidth": 2,
1876                 "pointSize": 5,
1877                 "scaleDistribution": {
1878                   "type": "linear"
1879                 },
1880                 "showPoints": "never",
1881                 "spanNulls": false,
1882                 "stacking": {
1883                   "group": "A",
1884                   "mode": "none"
1885                 },
1886                 "thresholdsStyle": {
1887                   "mode": "off"
1888                 }
1889               },
1890               "links": [],
1891               "mappings": [],
1892               "min": 0,
1893               "thresholds": {
1894                 "mode": "absolute",
1895                 "steps": [
1896                   {
1897                     "color": "green"
1898                   }
1899                 ]
1900               },
1901               "unit": "cps"
1902             },
1903             "overrides": []
1904           },
1905           "gridPos": {
1906             "h": 8,
1907             "w": 8,
1908             "x": 0,
1909             "y": 24
1910           },
1911           "id": 4,
1912           "links": [],
1913           "options": {
1914             "legend": {
1915               "calcs": [
1916                 "mean",
1917                 "lastNotNull",
1918                 "max"
1919               ],
1920               "displayMode": "table",
1921               "placement": "bottom",
1922               "showLegend": true
1923             },
1924             "tooltip": {
1925               "mode": "single"
1926             }
1927           },
1928           "pluginVersion": "8.1.3",
1929           "targets": [
1930             {
1931               "datasource": {
1932                 "type": "prometheus",
1933                 "uid": "PBFA97CFB590B2093"
1934               },
1935               "exemplar": true,
1936               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1937               "interval": "",
1938               "intervalFactor": 2,
1939               "legendFormat": "bytes/sec",
1940               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
1941               "refId": "A",
1942               "step": 4
1943             }
1944           ],
1945           "title": "Messages In",
1946           "type": "timeseries"
1947         },
1948         {
1949           "datasource": {
1950             "type": "prometheus",
1951             "uid": "PBFA97CFB590B2093"
1952           },
1953           "fieldConfig": {
1954             "defaults": {
1955               "color": {
1956                 "mode": "palette-classic"
1957               },
1958               "custom": {
1959                 "axisLabel": "Bytes/s",
1960                 "axisPlacement": "auto",
1961                 "barAlignment": 0,
1962                 "drawStyle": "line",
1963                 "fillOpacity": 10,
1964                 "gradientMode": "none",
1965                 "hideFrom": {
1966                   "legend": false,
1967                   "tooltip": false,
1968                   "viz": false
1969                 },
1970                 "lineInterpolation": "linear",
1971                 "lineWidth": 2,
1972                 "pointSize": 5,
1973                 "scaleDistribution": {
1974                   "type": "linear"
1975                 },
1976                 "showPoints": "never",
1977                 "spanNulls": false,
1978                 "stacking": {
1979                   "group": "A",
1980                   "mode": "none"
1981                 },
1982                 "thresholdsStyle": {
1983                   "mode": "off"
1984                 }
1985               },
1986               "links": [],
1987               "mappings": [],
1988               "min": 0,
1989               "thresholds": {
1990                 "mode": "absolute",
1991                 "steps": [
1992                   {
1993                     "color": "green"
1994                   },
1995                   {
1996                     "color": "red",
1997                     "value": 80
1998                   }
1999                 ]
2000               },
2001               "unit": "Bps"
2002             },
2003             "overrides": []
2004           },
2005           "gridPos": {
2006             "h": 8,
2007             "w": 8,
2008             "x": 8,
2009             "y": 24
2010           },
2011           "id": 5,
2012           "links": [],
2013           "options": {
2014             "legend": {
2015               "calcs": [
2016                 "mean",
2017                 "lastNotNull",
2018                 "max"
2019               ],
2020               "displayMode": "table",
2021               "placement": "bottom",
2022               "showLegend": true
2023             },
2024             "tooltip": {
2025               "mode": "single"
2026             }
2027           },
2028           "pluginVersion": "8.1.3",
2029           "targets": [
2030             {
2031               "datasource": {
2032                 "type": "prometheus",
2033                 "uid": "PBFA97CFB590B2093"
2034               },
2035               "exemplar": true,
2036               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2037               "format": "time_series",
2038               "interval": "",
2039               "intervalFactor": 2,
2040               "legendFormat": "bytes/sec",
2041               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2042               "refId": "A",
2043               "step": 4
2044             }
2045           ],
2046           "title": "Bytes In",
2047           "type": "timeseries"
2048         },
2049         {
2050           "datasource": {
2051             "type": "prometheus",
2052             "uid": "PBFA97CFB590B2093"
2053           },
2054           "fieldConfig": {
2055             "defaults": {
2056               "color": {
2057                 "mode": "palette-classic"
2058               },
2059               "custom": {
2060                 "axisLabel": "Bytes/s",
2061                 "axisPlacement": "auto",
2062                 "barAlignment": 0,
2063                 "drawStyle": "line",
2064                 "fillOpacity": 10,
2065                 "gradientMode": "none",
2066                 "hideFrom": {
2067                   "legend": false,
2068                   "tooltip": false,
2069                   "viz": false
2070                 },
2071                 "lineInterpolation": "linear",
2072                 "lineWidth": 2,
2073                 "pointSize": 5,
2074                 "scaleDistribution": {
2075                   "type": "linear"
2076                 },
2077                 "showPoints": "never",
2078                 "spanNulls": false,
2079                 "stacking": {
2080                   "group": "A",
2081                   "mode": "none"
2082                 },
2083                 "thresholdsStyle": {
2084                   "mode": "off"
2085                 }
2086               },
2087               "links": [],
2088               "mappings": [],
2089               "min": 0,
2090               "thresholds": {
2091                 "mode": "absolute",
2092                 "steps": [
2093                   {
2094                     "color": "green"
2095                   },
2096                   {
2097                     "color": "red",
2098                     "value": 80
2099                   }
2100                 ]
2101               },
2102               "unit": "Bps"
2103             },
2104             "overrides": []
2105           },
2106           "gridPos": {
2107             "h": 8,
2108             "w": 8,
2109             "x": 16,
2110             "y": 24
2111           },
2112           "id": 6,
2113           "links": [],
2114           "options": {
2115             "legend": {
2116               "calcs": [
2117                 "mean",
2118                 "lastNotNull",
2119                 "max"
2120               ],
2121               "displayMode": "table",
2122               "placement": "bottom",
2123               "showLegend": true
2124             },
2125             "tooltip": {
2126               "mode": "single"
2127             }
2128           },
2129           "pluginVersion": "8.1.3",
2130           "targets": [
2131             {
2132               "datasource": {
2133                 "type": "prometheus",
2134                 "uid": "PBFA97CFB590B2093"
2135               },
2136               "exemplar": true,
2137               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2138               "interval": "",
2139               "intervalFactor": 2,
2140               "legendFormat": "bytes/sec",
2141               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2142               "refId": "A",
2143               "step": 4
2144             }
2145           ],
2146           "title": "Bytes Out",
2147           "type": "timeseries"
2148         },
2149         {
2150           "datasource": {
2151             "type": "prometheus",
2152             "uid": "PBFA97CFB590B2093"
2153           },
2154           "fieldConfig": {
2155             "defaults": {
2156               "color": {
2157                 "mode": "palette-classic"
2158               },
2159               "custom": {
2160                 "axisLabel": "Messages/s",
2161                 "axisPlacement": "auto",
2162                 "barAlignment": 0,
2163                 "drawStyle": "line",
2164                 "fillOpacity": 10,
2165                 "gradientMode": "none",
2166                 "hideFrom": {
2167                   "legend": false,
2168                   "tooltip": false,
2169                   "viz": false
2170                 },
2171                 "lineInterpolation": "linear",
2172                 "lineWidth": 2,
2173                 "pointSize": 5,
2174                 "scaleDistribution": {
2175                   "type": "linear"
2176                 },
2177                 "showPoints": "never",
2178                 "spanNulls": false,
2179                 "stacking": {
2180                   "group": "A",
2181                   "mode": "none"
2182                 },
2183                 "thresholdsStyle": {
2184                   "mode": "off"
2185                 }
2186               },
2187               "links": [],
2188               "mappings": [],
2189               "thresholds": {
2190                 "mode": "absolute",
2191                 "steps": [
2192                   {
2193                     "color": "green"
2194                   },
2195                   {
2196                     "color": "red",
2197                     "value": 80
2198                   }
2199                 ]
2200               },
2201               "unit": "cps"
2202             },
2203             "overrides": []
2204           },
2205           "gridPos": {
2206             "h": 8,
2207             "w": 8,
2208             "x": 0,
2209             "y": 32
2210           },
2211           "id": 10,
2212           "links": [],
2213           "options": {
2214             "legend": {
2215               "calcs": [
2216                 "mean",
2217                 "lastNotNull",
2218                 "max"
2219               ],
2220               "displayMode": "table",
2221               "placement": "bottom",
2222               "showLegend": true
2223             },
2224             "tooltip": {
2225               "mode": "single"
2226             }
2227           },
2228           "pluginVersion": "8.1.3",
2229           "targets": [
2230             {
2231               "datasource": {
2232                 "type": "prometheus",
2233                 "uid": "PBFA97CFB590B2093"
2234               },
2235               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2236               "format": "time_series",
2237               "interval": "",
2238               "intervalFactor": 2,
2239               "legendFormat": "{{instance}}",
2240               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2241               "refId": "A",
2242               "step": 4
2243             }
2244           ],
2245           "title": "Messages In Per Broker",
2246           "type": "timeseries"
2247         },
2248         {
2249           "datasource": {
2250             "type": "prometheus",
2251             "uid": "PBFA97CFB590B2093"
2252           },
2253           "fieldConfig": {
2254             "defaults": {
2255               "color": {
2256                 "mode": "palette-classic"
2257               },
2258               "custom": {
2259                 "axisLabel": "Bytes/s",
2260                 "axisPlacement": "auto",
2261                 "barAlignment": 0,
2262                 "drawStyle": "line",
2263                 "fillOpacity": 10,
2264                 "gradientMode": "none",
2265                 "hideFrom": {
2266                   "legend": false,
2267                   "tooltip": false,
2268                   "viz": false
2269                 },
2270                 "lineInterpolation": "linear",
2271                 "lineWidth": 2,
2272                 "pointSize": 5,
2273                 "scaleDistribution": {
2274                   "type": "linear"
2275                 },
2276                 "showPoints": "never",
2277                 "spanNulls": false,
2278                 "stacking": {
2279                   "group": "A",
2280                   "mode": "none"
2281                 },
2282                 "thresholdsStyle": {
2283                   "mode": "off"
2284                 }
2285               },
2286               "links": [],
2287               "mappings": [],
2288               "thresholds": {
2289                 "mode": "absolute",
2290                 "steps": [
2291                   {
2292                     "color": "green"
2293                   },
2294                   {
2295                     "color": "red",
2296                     "value": 80
2297                   }
2298                 ]
2299               },
2300               "unit": "Bps"
2301             },
2302             "overrides": []
2303           },
2304           "gridPos": {
2305             "h": 8,
2306             "w": 8,
2307             "x": 8,
2308             "y": 32
2309           },
2310           "id": 7,
2311           "links": [],
2312           "options": {
2313             "legend": {
2314               "calcs": [
2315                 "mean",
2316                 "lastNotNull",
2317                 "max"
2318               ],
2319               "displayMode": "table",
2320               "placement": "bottom",
2321               "showLegend": true
2322             },
2323             "tooltip": {
2324               "mode": "single"
2325             }
2326           },
2327           "pluginVersion": "8.1.3",
2328           "targets": [
2329             {
2330               "datasource": {
2331                 "type": "prometheus",
2332                 "uid": "PBFA97CFB590B2093"
2333               },
2334               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2335               "format": "time_series",
2336               "interval": "",
2337               "intervalFactor": 2,
2338               "legendFormat": "{{instance}}",
2339               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2340               "refId": "A",
2341               "step": 4
2342             }
2343           ],
2344           "title": "Bytes In Per Broker",
2345           "type": "timeseries"
2346         },
2347         {
2348           "datasource": {
2349             "type": "prometheus",
2350             "uid": "PBFA97CFB590B2093"
2351           },
2352           "fieldConfig": {
2353             "defaults": {
2354               "color": {
2355                 "mode": "palette-classic"
2356               },
2357               "custom": {
2358                 "axisLabel": "",
2359                 "axisPlacement": "auto",
2360                 "barAlignment": 0,
2361                 "drawStyle": "line",
2362                 "fillOpacity": 10,
2363                 "gradientMode": "none",
2364                 "hideFrom": {
2365                   "legend": false,
2366                   "tooltip": false,
2367                   "viz": false
2368                 },
2369                 "lineInterpolation": "linear",
2370                 "lineWidth": 1,
2371                 "pointSize": 5,
2372                 "scaleDistribution": {
2373                   "type": "linear"
2374                 },
2375                 "showPoints": "never",
2376                 "spanNulls": true,
2377                 "stacking": {
2378                   "group": "A",
2379                   "mode": "none"
2380                 },
2381                 "thresholdsStyle": {
2382                   "mode": "off"
2383                 }
2384               },
2385               "links": [],
2386               "mappings": [],
2387               "min": 0,
2388               "thresholds": {
2389                 "mode": "absolute",
2390                 "steps": [
2391                   {
2392                     "color": "green"
2393                   },
2394                   {
2395                     "color": "red",
2396                     "value": 80
2397                   }
2398                 ]
2399               },
2400               "unit": "binBps"
2401             },
2402             "overrides": []
2403           },
2404           "gridPos": {
2405             "h": 8,
2406             "w": 8,
2407             "x": 16,
2408             "y": 32
2409           },
2410           "id": 9,
2411           "links": [],
2412           "options": {
2413             "legend": {
2414               "calcs": [
2415                 "mean",
2416                 "lastNotNull",
2417                 "max"
2418               ],
2419               "displayMode": "table",
2420               "placement": "bottom",
2421               "showLegend": true
2422             },
2423             "tooltip": {
2424               "mode": "single"
2425             }
2426           },
2427           "pluginVersion": "8.1.3",
2428           "targets": [
2429             {
2430               "datasource": {
2431                 "type": "prometheus",
2432                 "uid": "PBFA97CFB590B2093"
2433               },
2434               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2435               "format": "time_series",
2436               "interval": "",
2437               "intervalFactor": 1,
2438               "legendFormat": "{{instance}}",
2439               "refId": "A"
2440             }
2441           ],
2442           "title": "Bytes Out Per Broker",
2443           "type": "timeseries"
2444         }
2445       ],
2446       "targets": [
2447         {
2448           "datasource": {
2449             "type": "datasource",
2450             "uid": "grafana"
2451           },
2452           "refId": "A"
2453         }
2454       ],
2455       "title": "Throughput In/Out",
2456       "type": "row"
2457     },
2458     {
2459       "collapsed": true,
2460       "datasource": {
2461         "type": "datasource",
2462         "uid": "grafana"
2463       },
2464       "gridPos": {
2465         "h": 1,
2466         "w": 24,
2467         "x": 0,
2468         "y": 36
2469       },
2470       "id": 44,
2471       "panels": [
2472         {
2473           "datasource": {
2474             "type": "prometheus",
2475             "uid": "PBFA97CFB590B2093"
2476           },
2477           "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2478           "fieldConfig": {
2479             "defaults": {
2480               "color": {
2481                 "mode": "palette-classic"
2482               },
2483               "custom": {
2484                 "axisLabel": "",
2485                 "axisPlacement": "auto",
2486                 "barAlignment": 0,
2487                 "drawStyle": "line",
2488                 "fillOpacity": 10,
2489                 "gradientMode": "none",
2490                 "hideFrom": {
2491                   "legend": false,
2492                   "tooltip": false,
2493                   "viz": false
2494                 },
2495                 "lineInterpolation": "linear",
2496                 "lineWidth": 1,
2497                 "pointSize": 5,
2498                 "scaleDistribution": {
2499                   "type": "linear"
2500                 },
2501                 "showPoints": "never",
2502                 "spanNulls": true,
2503                 "stacking": {
2504                   "group": "A",
2505                   "mode": "none"
2506                 },
2507                 "thresholdsStyle": {
2508                   "mode": "off"
2509                 }
2510               },
2511               "links": [],
2512               "mappings": [],
2513               "thresholds": {
2514                 "mode": "absolute",
2515                 "steps": [
2516                   {
2517                     "color": "green"
2518                   },
2519                   {
2520                     "color": "red",
2521                     "value": 80
2522                   }
2523                 ]
2524               },
2525               "unit": "percentunit"
2526             },
2527             "overrides": []
2528           },
2529           "gridPos": {
2530             "h": 9,
2531             "w": 12,
2532             "x": 0,
2533             "y": 25
2534           },
2535           "id": 24,
2536           "options": {
2537             "legend": {
2538               "calcs": [
2539                 "mean",
2540                 "lastNotNull",
2541                 "max"
2542               ],
2543               "displayMode": "table",
2544               "placement": "bottom",
2545               "showLegend": true
2546             },
2547             "tooltip": {
2548               "mode": "single"
2549             }
2550           },
2551           "pluginVersion": "8.1.3",
2552           "targets": [
2553             {
2554               "datasource": {
2555                 "type": "prometheus",
2556                 "uid": "PBFA97CFB590B2093"
2557               },
2558               "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2559               "interval": "",
2560               "legendFormat": "{{instance}}",
2561               "refId": "A"
2562             }
2563           ],
2564           "title": "Network Processor Avg Usage Percent",
2565           "type": "timeseries"
2566         },
2567         {
2568           "datasource": {
2569             "type": "prometheus",
2570             "uid": "PBFA97CFB590B2093"
2571           },
2572           "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2573           "fieldConfig": {
2574             "defaults": {
2575               "color": {
2576                 "mode": "palette-classic"
2577               },
2578               "custom": {
2579                 "axisLabel": "",
2580                 "axisPlacement": "auto",
2581                 "barAlignment": 0,
2582                 "drawStyle": "line",
2583                 "fillOpacity": 10,
2584                 "gradientMode": "none",
2585                 "hideFrom": {
2586                   "legend": false,
2587                   "tooltip": false,
2588                   "viz": false
2589                 },
2590                 "lineInterpolation": "linear",
2591                 "lineWidth": 1,
2592                 "pointSize": 5,
2593                 "scaleDistribution": {
2594                   "type": "linear"
2595                 },
2596                 "showPoints": "never",
2597                 "spanNulls": true,
2598                 "stacking": {
2599                   "group": "A",
2600                   "mode": "none"
2601                 },
2602                 "thresholdsStyle": {
2603                   "mode": "off"
2604                 }
2605               },
2606               "links": [],
2607               "mappings": [],
2608               "min": 0,
2609               "thresholds": {
2610                 "mode": "absolute",
2611                 "steps": [
2612                   {
2613                     "color": "green"
2614                   },
2615                   {
2616                     "color": "red",
2617                     "value": 80
2618                   }
2619                 ]
2620               },
2621               "unit": "percentunit"
2622             },
2623             "overrides": []
2624           },
2625           "gridPos": {
2626             "h": 9,
2627             "w": 12,
2628             "x": 12,
2629             "y": 25
2630           },
2631           "id": 25,
2632           "options": {
2633             "legend": {
2634               "calcs": [
2635                 "mean",
2636                 "lastNotNull",
2637                 "max"
2638               ],
2639               "displayMode": "table",
2640               "placement": "bottom",
2641               "showLegend": true
2642             },
2643             "tooltip": {
2644               "mode": "single"
2645             }
2646           },
2647           "pluginVersion": "8.1.3",
2648           "targets": [
2649             {
2650               "datasource": {
2651                 "type": "prometheus",
2652                 "uid": "PBFA97CFB590B2093"
2653               },
2654               "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2655               "interval": "",
2656               "legendFormat": "{{instance}}",
2657               "refId": "A"
2658             }
2659           ],
2660           "title": "Request Handler Avg Percent",
2661           "type": "timeseries"
2662         }
2663       ],
2664       "targets": [
2665         {
2666           "datasource": {
2667             "type": "datasource",
2668             "uid": "grafana"
2669           },
2670           "refId": "A"
2671         }
2672       ],
2673       "title": "Thread utilization",
2674       "type": "row"
2675     },
2676     {
2677       "collapsed": true,
2678       "datasource": {
2679         "type": "datasource",
2680         "uid": "grafana"
2681       },
2682       "gridPos": {
2683         "h": 1,
2684         "w": 24,
2685         "x": 0,
2686         "y": 37
2687       },
2688       "id": 86,
2689       "panels": [
2690         {
2691           "datasource": {
2692             "type": "prometheus",
2693             "uid": "PBFA97CFB590B2093"
2694           },
2695           "description": "Latency in millseconds for ZooKeeper requests from broker.\n",
2696           "fieldConfig": {
2697             "defaults": {
2698               "color": {
2699                 "mode": "palette-classic"
2700               },
2701               "custom": {
2702                 "axisLabel": "",
2703                 "axisPlacement": "auto",
2704                 "barAlignment": 0,
2705                 "drawStyle": "line",
2706                 "fillOpacity": 10,
2707                 "gradientMode": "none",
2708                 "hideFrom": {
2709                   "legend": false,
2710                   "tooltip": false,
2711                   "viz": false
2712                 },
2713                 "lineInterpolation": "linear",
2714                 "lineWidth": 1,
2715                 "pointSize": 5,
2716                 "scaleDistribution": {
2717                   "type": "linear"
2718                 },
2719                 "showPoints": "never",
2720                 "spanNulls": true,
2721                 "stacking": {
2722                   "group": "A",
2723                   "mode": "none"
2724                 },
2725                 "thresholdsStyle": {
2726                   "mode": "off"
2727                 }
2728               },
2729               "links": [],
2730               "mappings": [],
2731               "thresholds": {
2732                 "mode": "absolute",
2733                 "steps": [
2734                   {
2735                     "color": "green"
2736                   },
2737                   {
2738                     "color": "red",
2739                     "value": 80
2740                   }
2741                 ]
2742               },
2743               "unit": "ms"
2744             },
2745             "overrides": []
2746           },
2747           "gridPos": {
2748             "h": 9,
2749             "w": 12,
2750             "x": 0,
2751             "y": 26
2752           },
2753           "id": 88,
2754           "options": {
2755             "legend": {
2756               "calcs": [
2757                 "mean",
2758                 "lastNotNull",
2759                 "max"
2760               ],
2761               "displayMode": "table",
2762               "placement": "bottom",
2763               "showLegend": true
2764             },
2765             "tooltip": {
2766               "mode": "single"
2767             }
2768           },
2769           "pluginVersion": "8.1.3",
2770           "targets": [
2771             {
2772               "datasource": {
2773                 "type": "prometheus",
2774                 "uid": "PBFA97CFB590B2093"
2775               },
2776               "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}",
2777               "hide": false,
2778               "legendFormat": "{{instance}} - {{quantile}}",
2779               "refId": "A"
2780             }
2781           ],
2782           "title": "Zookeeper Request Latency",
2783           "type": "timeseries"
2784         },
2785         {
2786           "datasource": {
2787             "type": "prometheus",
2788             "uid": "PBFA97CFB590B2093"
2789           },
2790           "description": "",
2791           "fieldConfig": {
2792             "defaults": {
2793               "color": {
2794                 "mode": "palette-classic"
2795               },
2796               "custom": {
2797                 "axisLabel": "",
2798                 "axisPlacement": "auto",
2799                 "barAlignment": 0,
2800                 "drawStyle": "line",
2801                 "fillOpacity": 10,
2802                 "gradientMode": "none",
2803                 "hideFrom": {
2804                   "legend": false,
2805                   "tooltip": false,
2806                   "viz": false
2807                 },
2808                 "lineInterpolation": "linear",
2809                 "lineWidth": 1,
2810                 "pointSize": 5,
2811                 "scaleDistribution": {
2812                   "type": "linear"
2813                 },
2814                 "showPoints": "never",
2815                 "spanNulls": true,
2816                 "stacking": {
2817                   "group": "A",
2818                   "mode": "none"
2819                 },
2820                 "thresholdsStyle": {
2821                   "mode": "off"
2822                 }
2823               },
2824               "links": [],
2825               "mappings": [],
2826               "min": 0,
2827               "thresholds": {
2828                 "mode": "absolute",
2829                 "steps": [
2830                   {
2831                     "color": "green"
2832                   },
2833                   {
2834                     "color": "red",
2835                     "value": 80
2836                   }
2837                 ]
2838               },
2839               "unit": "short"
2840             },
2841             "overrides": []
2842           },
2843           "gridPos": {
2844             "h": 9,
2845             "w": 12,
2846             "x": 12,
2847             "y": 26
2848           },
2849           "id": 92,
2850           "options": {
2851             "legend": {
2852               "calcs": [
2853                 "mean",
2854                 "lastNotNull",
2855                 "max"
2856               ],
2857               "displayMode": "table",
2858               "placement": "bottom",
2859               "showLegend": true
2860             },
2861             "tooltip": {
2862               "mode": "single"
2863             }
2864           },
2865           "pluginVersion": "8.1.3",
2866           "targets": [
2867             {
2868               "datasource": {
2869                 "type": "prometheus",
2870                 "uid": "PBFA97CFB590B2093"
2871               },
2872               "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2873               "hide": false,
2874               "instant": false,
2875               "interval": "",
2876               "legendFormat": "{{instance}}",
2877               "refId": "A"
2878             }
2879           ],
2880           "title": "Zookeeper connections per sec",
2881           "type": "timeseries"
2882         },
2883         {
2884           "datasource": {
2885             "type": "prometheus",
2886             "uid": "PBFA97CFB590B2093"
2887           },
2888           "description": "",
2889           "fieldConfig": {
2890             "defaults": {
2891               "color": {
2892                 "mode": "palette-classic"
2893               },
2894               "custom": {
2895                 "axisLabel": "",
2896                 "axisPlacement": "auto",
2897                 "barAlignment": 0,
2898                 "drawStyle": "line",
2899                 "fillOpacity": 10,
2900                 "gradientMode": "none",
2901                 "hideFrom": {
2902                   "legend": false,
2903                   "tooltip": false,
2904                   "viz": false
2905                 },
2906                 "lineInterpolation": "linear",
2907                 "lineWidth": 1,
2908                 "pointSize": 5,
2909                 "scaleDistribution": {
2910                   "type": "linear"
2911                 },
2912                 "showPoints": "never",
2913                 "spanNulls": true,
2914                 "stacking": {
2915                   "group": "A",
2916                   "mode": "none"
2917                 },
2918                 "thresholdsStyle": {
2919                   "mode": "off"
2920                 }
2921               },
2922               "links": [],
2923               "mappings": [],
2924               "min": 0,
2925               "thresholds": {
2926                 "mode": "absolute",
2927                 "steps": [
2928                   {
2929                     "color": "green"
2930                   },
2931                   {
2932                     "color": "red",
2933                     "value": 80
2934                   }
2935                 ]
2936               },
2937               "unit": "short"
2938             },
2939             "overrides": []
2940           },
2941           "gridPos": {
2942             "h": 9,
2943             "w": 8,
2944             "x": 0,
2945             "y": 35
2946           },
2947           "id": 89,
2948           "options": {
2949             "legend": {
2950               "calcs": [
2951                 "mean",
2952                 "lastNotNull",
2953                 "max"
2954               ],
2955               "displayMode": "table",
2956               "placement": "bottom",
2957               "showLegend": true
2958             },
2959             "tooltip": {
2960               "mode": "single"
2961             }
2962           },
2963           "pluginVersion": "8.1.3",
2964           "targets": [
2965             {
2966               "datasource": {
2967                 "type": "prometheus",
2968                 "uid": "PBFA97CFB590B2093"
2969               },
2970               "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2971               "hide": false,
2972               "interval": "",
2973               "legendFormat": "{{instance}}",
2974               "refId": "A"
2975             }
2976           ],
2977           "title": "Zookeeper expired connections per sec",
2978           "type": "timeseries"
2979         },
2980         {
2981           "datasource": {
2982             "type": "prometheus",
2983             "uid": "PBFA97CFB590B2093"
2984           },
2985           "description": "",
2986           "fieldConfig": {
2987             "defaults": {
2988               "color": {
2989                 "mode": "palette-classic"
2990               },
2991               "custom": {
2992                 "axisLabel": "",
2993                 "axisPlacement": "auto",
2994                 "barAlignment": 0,
2995                 "drawStyle": "line",
2996                 "fillOpacity": 10,
2997                 "gradientMode": "none",
2998                 "hideFrom": {
2999                   "legend": false,
3000                   "tooltip": false,
3001                   "viz": false
3002                 },
3003                 "lineInterpolation": "linear",
3004                 "lineWidth": 1,
3005                 "pointSize": 5,
3006                 "scaleDistribution": {
3007                   "type": "linear"
3008                 },
3009                 "showPoints": "never",
3010                 "spanNulls": true,
3011                 "stacking": {
3012                   "group": "A",
3013                   "mode": "none"
3014                 },
3015                 "thresholdsStyle": {
3016                   "mode": "off"
3017                 }
3018               },
3019               "links": [],
3020               "mappings": [],
3021               "min": 0,
3022               "thresholds": {
3023                 "mode": "absolute",
3024                 "steps": [
3025                   {
3026                     "color": "green"
3027                   },
3028                   {
3029                     "color": "red",
3030                     "value": 80
3031                   }
3032                 ]
3033               },
3034               "unit": "short"
3035             },
3036             "overrides": []
3037           },
3038           "gridPos": {
3039             "h": 9,
3040             "w": 8,
3041             "x": 8,
3042             "y": 35
3043           },
3044           "id": 90,
3045           "options": {
3046             "legend": {
3047               "calcs": [
3048                 "mean",
3049                 "lastNotNull",
3050                 "max"
3051               ],
3052               "displayMode": "table",
3053               "placement": "bottom",
3054               "showLegend": true
3055             },
3056             "tooltip": {
3057               "mode": "single"
3058             }
3059           },
3060           "pluginVersion": "8.1.3",
3061           "targets": [
3062             {
3063               "datasource": {
3064                 "type": "prometheus",
3065                 "uid": "PBFA97CFB590B2093"
3066               },
3067               "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
3068               "hide": false,
3069               "interval": "",
3070               "legendFormat": "{{instance}}",
3071               "refId": "A"
3072             }
3073           ],
3074           "title": "Zookeeper disconnect per sec",
3075           "type": "timeseries"
3076         },
3077         {
3078           "datasource": {
3079             "type": "prometheus",
3080             "uid": "PBFA97CFB590B2093"
3081           },
3082           "description": "",
3083           "fieldConfig": {
3084             "defaults": {
3085               "color": {
3086                 "mode": "palette-classic"
3087               },
3088               "custom": {
3089                 "axisLabel": "",
3090                 "axisPlacement": "auto",
3091                 "barAlignment": 0,
3092                 "drawStyle": "line",
3093                 "fillOpacity": 10,
3094                 "gradientMode": "none",
3095                 "hideFrom": {
3096                   "legend": false,
3097                   "tooltip": false,
3098                   "viz": false
3099                 },
3100                 "lineInterpolation": "linear",
3101                 "lineWidth": 1,
3102                 "pointSize": 5,
3103                 "scaleDistribution": {
3104                   "type": "linear"
3105                 },
3106                 "showPoints": "never",
3107                 "spanNulls": true,
3108                 "stacking": {
3109                   "group": "A",
3110                   "mode": "none"
3111                 },
3112                 "thresholdsStyle": {
3113                   "mode": "off"
3114                 }
3115               },
3116               "links": [],
3117               "mappings": [],
3118               "min": 0,
3119               "thresholds": {
3120                 "mode": "absolute",
3121                 "steps": [
3122                   {
3123                     "color": "green"
3124                   },
3125                   {
3126                     "color": "red",
3127                     "value": 80
3128                   }
3129                 ]
3130               },
3131               "unit": "short"
3132             },
3133             "overrides": []
3134           },
3135           "gridPos": {
3136             "h": 9,
3137             "w": 8,
3138             "x": 16,
3139             "y": 35
3140           },
3141           "id": 91,
3142           "options": {
3143             "legend": {
3144               "calcs": [
3145                 "mean",
3146                 "lastNotNull",
3147                 "max"
3148               ],
3149               "displayMode": "table",
3150               "placement": "bottom",
3151               "showLegend": true
3152             },
3153             "tooltip": {
3154               "mode": "single"
3155             }
3156           },
3157           "pluginVersion": "8.1.3",
3158           "targets": [
3159             {
3160               "datasource": {
3161                 "type": "prometheus",
3162                 "uid": "PBFA97CFB590B2093"
3163               },
3164               "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
3165               "hide": false,
3166               "interval": "",
3167               "legendFormat": "{{instance}}",
3168               "refId": "A"
3169             }
3170           ],
3171           "title": "Zookeeper auth failures per sec",
3172           "type": "timeseries"
3173         }
3174       ],
3175       "targets": [
3176         {
3177           "datasource": {
3178             "type": "datasource",
3179             "uid": "grafana"
3180           },
3181           "refId": "A"
3182         }
3183       ],
3184       "title": "Zookeeper",
3185       "type": "row"
3186     },
3187     {
3188       "collapsed": false,
3189       "datasource": {
3190         "type": "datasource",
3191         "uid": "grafana"
3192       },
3193       "gridPos": {
3194         "h": 1,
3195         "w": 24,
3196         "x": 0,
3197         "y": 38
3198       },
3199       "id": 82,
3200       "panels": [],
3201       "targets": [
3202         {
3203           "datasource": {
3204             "type": "datasource",
3205             "uid": "grafana"
3206           },
3207           "refId": "A"
3208         }
3209       ],
3210       "title": "Isr Shrinks / Expands",
3211       "type": "row"
3212     },
3213     {
3214       "datasource": {
3215         "type": "prometheus",
3216         "uid": "PBFA97CFB590B2093"
3217       },
3218       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
3219       "fieldConfig": {
3220         "defaults": {
3221           "color": {
3222             "mode": "palette-classic"
3223           },
3224           "custom": {
3225             "axisCenteredZero": false,
3226             "axisColorMode": "text",
3227             "axisLabel": "",
3228             "axisPlacement": "auto",
3229             "barAlignment": 0,
3230             "drawStyle": "line",
3231             "fillOpacity": 10,
3232             "gradientMode": "none",
3233             "hideFrom": {
3234               "legend": false,
3235               "tooltip": false,
3236               "viz": false
3237             },
3238             "lineInterpolation": "linear",
3239             "lineWidth": 1,
3240             "pointSize": 5,
3241             "scaleDistribution": {
3242               "type": "linear"
3243             },
3244             "showPoints": "never",
3245             "spanNulls": true,
3246             "stacking": {
3247               "group": "A",
3248               "mode": "none"
3249             },
3250             "thresholdsStyle": {
3251               "mode": "off"
3252             }
3253           },
3254           "links": [],
3255           "mappings": [],
3256           "min": 0,
3257           "thresholds": {
3258             "mode": "absolute",
3259             "steps": [
3260               {
3261                 "color": "green",
3262                 "value": null
3263               },
3264               {
3265                 "color": "red",
3266                 "value": 80
3267               }
3268             ]
3269           },
3270           "unit": "ops"
3271         },
3272         "overrides": []
3273       },
3274       "gridPos": {
3275         "h": 9,
3276         "w": 12,
3277         "x": 0,
3278         "y": 39
3279       },
3280       "id": 80,
3281       "options": {
3282         "legend": {
3283           "calcs": [
3284             "mean",
3285             "lastNotNull",
3286             "max"
3287           ],
3288           "displayMode": "table",
3289           "placement": "bottom",
3290           "showLegend": true
3291         },
3292         "tooltip": {
3293           "mode": "single",
3294           "sort": "none"
3295         }
3296       },
3297       "pluginVersion": "8.1.3",
3298       "targets": [
3299         {
3300           "datasource": {
3301             "type": "prometheus",
3302             "uid": "PBFA97CFB590B2093"
3303           },
3304           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3305           "interval": "",
3306           "legendFormat": "{{instance}}",
3307           "refId": "A"
3308         }
3309       ],
3310       "title": "IsrShrinks per Sec",
3311       "type": "timeseries"
3312     },
3313     {
3314       "datasource": {
3315         "type": "prometheus",
3316         "uid": "PBFA97CFB590B2093"
3317       },
3318       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
3319       "fieldConfig": {
3320         "defaults": {
3321           "color": {
3322             "mode": "palette-classic"
3323           },
3324           "custom": {
3325             "axisCenteredZero": false,
3326             "axisColorMode": "text",
3327             "axisLabel": "",
3328             "axisPlacement": "auto",
3329             "barAlignment": 0,
3330             "drawStyle": "line",
3331             "fillOpacity": 10,
3332             "gradientMode": "none",
3333             "hideFrom": {
3334               "legend": false,
3335               "tooltip": false,
3336               "viz": false
3337             },
3338             "lineInterpolation": "linear",
3339             "lineWidth": 1,
3340             "pointSize": 5,
3341             "scaleDistribution": {
3342               "type": "linear"
3343             },
3344             "showPoints": "never",
3345             "spanNulls": true,
3346             "stacking": {
3347               "group": "A",
3348               "mode": "none"
3349             },
3350             "thresholdsStyle": {
3351               "mode": "off"
3352             }
3353           },
3354           "links": [],
3355           "mappings": [],
3356           "min": 0,
3357           "thresholds": {
3358             "mode": "absolute",
3359             "steps": [
3360               {
3361                 "color": "green",
3362                 "value": null
3363               },
3364               {
3365                 "color": "red",
3366                 "value": 80
3367               }
3368             ]
3369           },
3370           "unit": "ops"
3371         },
3372         "overrides": []
3373       },
3374       "gridPos": {
3375         "h": 9,
3376         "w": 12,
3377         "x": 12,
3378         "y": 39
3379       },
3380       "id": 83,
3381       "options": {
3382         "legend": {
3383           "calcs": [
3384             "mean",
3385             "lastNotNull",
3386             "max"
3387           ],
3388           "displayMode": "table",
3389           "placement": "bottom",
3390           "showLegend": true
3391         },
3392         "tooltip": {
3393           "mode": "single",
3394           "sort": "none"
3395         }
3396       },
3397       "pluginVersion": "8.1.3",
3398       "targets": [
3399         {
3400           "datasource": {
3401             "type": "prometheus",
3402             "uid": "PBFA97CFB590B2093"
3403           },
3404           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3405           "hide": false,
3406           "interval": "",
3407           "legendFormat": "{{instance}}",
3408           "refId": "A"
3409         }
3410       ],
3411       "title": "IsrExpands per Sec",
3412       "type": "timeseries"
3413     },
3414     {
3415       "collapsed": true,
3416       "datasource": {
3417         "type": "datasource",
3418         "uid": "grafana"
3419       },
3420       "gridPos": {
3421         "h": 1,
3422         "w": 24,
3423         "x": 0,
3424         "y": 48
3425       },
3426       "id": 53,
3427       "panels": [
3428         {
3429           "datasource": {
3430             "type": "prometheus",
3431             "uid": "PBFA97CFB590B2093"
3432           },
3433           "fieldConfig": {
3434             "defaults": {
3435               "color": {
3436                 "mode": "palette-classic"
3437               },
3438               "custom": {
3439                 "axisLabel": "",
3440                 "axisPlacement": "auto",
3441                 "barAlignment": 0,
3442                 "drawStyle": "line",
3443                 "fillOpacity": 10,
3444                 "gradientMode": "none",
3445                 "hideFrom": {
3446                   "legend": false,
3447                   "tooltip": false,
3448                   "viz": false
3449                 },
3450                 "lineInterpolation": "linear",
3451                 "lineWidth": 1,
3452                 "pointSize": 5,
3453                 "scaleDistribution": {
3454                   "type": "linear"
3455                 },
3456                 "showPoints": "never",
3457                 "spanNulls": true,
3458                 "stacking": {
3459                   "group": "A",
3460                   "mode": "none"
3461                 },
3462                 "thresholdsStyle": {
3463                   "mode": "off"
3464                 }
3465               },
3466               "links": [],
3467               "mappings": [],
3468               "thresholds": {
3469                 "mode": "absolute",
3470                 "steps": [
3471                   {
3472                     "color": "green"
3473                   },
3474                   {
3475                     "color": "red",
3476                     "value": 80
3477                   }
3478                 ]
3479               },
3480               "unit": "bytes"
3481             },
3482             "overrides": []
3483           },
3484           "gridPos": {
3485             "h": 9,
3486             "w": 12,
3487             "x": 0,
3488             "y": 28
3489           },
3490           "id": 55,
3491           "options": {
3492             "legend": {
3493               "calcs": [
3494                 "mean",
3495                 "lastNotNull",
3496                 "max"
3497               ],
3498               "displayMode": "table",
3499               "placement": "bottom",
3500               "showLegend": true
3501             },
3502             "tooltip": {
3503               "mode": "single"
3504             }
3505           },
3506           "pluginVersion": "8.1.3",
3507           "targets": [
3508             {
3509               "datasource": {
3510                 "type": "prometheus",
3511                 "uid": "PBFA97CFB590B2093"
3512               },
3513               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3514               "legendFormat": "{{topic}}",
3515               "refId": "A"
3516             }
3517           ],
3518           "title": "Log size per Topic",
3519           "type": "timeseries"
3520         },
3521         {
3522           "datasource": {
3523             "type": "prometheus",
3524             "uid": "PBFA97CFB590B2093"
3525           },
3526           "fieldConfig": {
3527             "defaults": {
3528               "color": {
3529                 "mode": "palette-classic"
3530               },
3531               "custom": {
3532                 "axisLabel": "",
3533                 "axisPlacement": "auto",
3534                 "barAlignment": 0,
3535                 "drawStyle": "line",
3536                 "fillOpacity": 10,
3537                 "gradientMode": "none",
3538                 "hideFrom": {
3539                   "legend": false,
3540                   "tooltip": false,
3541                   "viz": false
3542                 },
3543                 "lineInterpolation": "linear",
3544                 "lineWidth": 1,
3545                 "pointSize": 5,
3546                 "scaleDistribution": {
3547                   "type": "linear"
3548                 },
3549                 "showPoints": "never",
3550                 "spanNulls": true,
3551                 "stacking": {
3552                   "group": "A",
3553                   "mode": "none"
3554                 },
3555                 "thresholdsStyle": {
3556                   "mode": "off"
3557                 }
3558               },
3559               "links": [],
3560               "mappings": [],
3561               "thresholds": {
3562                 "mode": "absolute",
3563                 "steps": [
3564                   {
3565                     "color": "green"
3566                   },
3567                   {
3568                     "color": "red",
3569                     "value": 80
3570                   }
3571                 ]
3572               },
3573               "unit": "bytes"
3574             },
3575             "overrides": []
3576           },
3577           "gridPos": {
3578             "h": 9,
3579             "w": 12,
3580             "x": 12,
3581             "y": 28
3582           },
3583           "id": 56,
3584           "options": {
3585             "legend": {
3586               "calcs": [
3587                 "mean",
3588                 "lastNotNull",
3589                 "max"
3590               ],
3591               "displayMode": "table",
3592               "placement": "bottom",
3593               "showLegend": true
3594             },
3595             "tooltip": {
3596               "mode": "single"
3597             }
3598           },
3599           "pluginVersion": "8.1.3",
3600           "targets": [
3601             {
3602               "datasource": {
3603                 "type": "prometheus",
3604                 "uid": "PBFA97CFB590B2093"
3605               },
3606               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3607               "legendFormat": "{{instance}}",
3608               "refId": "A"
3609             }
3610           ],
3611           "title": "Log size per Broker",
3612           "type": "timeseries"
3613         }
3614       ],
3615       "targets": [
3616         {
3617           "datasource": {
3618             "type": "datasource",
3619             "uid": "grafana"
3620           },
3621           "refId": "A"
3622         }
3623       ],
3624       "title": "Logs size",
3625       "type": "row"
3626     },
3627     {
3628       "collapsed": true,
3629       "datasource": {
3630         "type": "datasource",
3631         "uid": "grafana"
3632       },
3633       "gridPos": {
3634         "h": 1,
3635         "w": 24,
3636         "x": 0,
3637         "y": 49
3638       },
3639       "id": 58,
3640       "panels": [
3641         {
3642           "datasource": {
3643             "type": "prometheus",
3644             "uid": "PBFA97CFB590B2093"
3645           },
3646           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3647           "fieldConfig": {
3648             "defaults": {
3649               "color": {
3650                 "mode": "palette-classic"
3651               },
3652               "custom": {
3653                 "axisLabel": "",
3654                 "axisPlacement": "auto",
3655                 "barAlignment": 0,
3656                 "drawStyle": "line",
3657                 "fillOpacity": 10,
3658                 "gradientMode": "none",
3659                 "hideFrom": {
3660                   "legend": false,
3661                   "tooltip": false,
3662                   "viz": false
3663                 },
3664                 "lineInterpolation": "linear",
3665                 "lineWidth": 1,
3666                 "pointSize": 5,
3667                 "scaleDistribution": {
3668                   "type": "linear"
3669                 },
3670                 "showPoints": "never",
3671                 "spanNulls": true,
3672                 "stacking": {
3673                   "group": "A",
3674                   "mode": "none"
3675                 },
3676                 "thresholdsStyle": {
3677                   "mode": "off"
3678                 }
3679               },
3680               "links": [],
3681               "mappings": [],
3682               "thresholds": {
3683                 "mode": "absolute",
3684                 "steps": [
3685                   {
3686                     "color": "green"
3687                   },
3688                   {
3689                     "color": "red",
3690                     "value": 80
3691                   }
3692                 ]
3693               },
3694               "unit": "ms"
3695             },
3696             "overrides": []
3697           },
3698           "gridPos": {
3699             "h": 9,
3700             "w": 12,
3701             "x": 0,
3702             "y": 29
3703           },
3704           "id": 60,
3705           "options": {
3706             "legend": {
3707               "calcs": [
3708                 "mean",
3709                 "lastNotNull",
3710                 "max"
3711               ],
3712               "displayMode": "list",
3713               "placement": "bottom",
3714               "showLegend": true
3715             },
3716             "tooltip": {
3717               "mode": "single"
3718             }
3719           },
3720           "pluginVersion": "8.1.3",
3721           "targets": [
3722             {
3723               "datasource": {
3724                 "type": "prometheus",
3725                 "uid": "PBFA97CFB590B2093"
3726               },
3727               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3728               "hide": false,
3729               "legendFormat": "{{instance}} - {{quantile}}",
3730               "refId": "A"
3731             }
3732           ],
3733           "title": "Producer - RequestQueueTimeMs",
3734           "type": "timeseries"
3735         },
3736         {
3737           "datasource": {
3738             "type": "prometheus",
3739             "uid": "PBFA97CFB590B2093"
3740           },
3741           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3742           "fieldConfig": {
3743             "defaults": {
3744               "color": {
3745                 "mode": "palette-classic"
3746               },
3747               "custom": {
3748                 "axisLabel": "",
3749                 "axisPlacement": "auto",
3750                 "barAlignment": 0,
3751                 "drawStyle": "line",
3752                 "fillOpacity": 10,
3753                 "gradientMode": "none",
3754                 "hideFrom": {
3755                   "legend": false,
3756                   "tooltip": false,
3757                   "viz": false
3758                 },
3759                 "lineInterpolation": "linear",
3760                 "lineWidth": 1,
3761                 "pointSize": 5,
3762                 "scaleDistribution": {
3763                   "type": "linear"
3764                 },
3765                 "showPoints": "never",
3766                 "spanNulls": true,
3767                 "stacking": {
3768                   "group": "A",
3769                   "mode": "none"
3770                 },
3771                 "thresholdsStyle": {
3772                   "mode": "off"
3773                 }
3774               },
3775               "links": [],
3776               "mappings": [],
3777               "thresholds": {
3778                 "mode": "absolute",
3779                 "steps": [
3780                   {
3781                     "color": "green"
3782                   },
3783                   {
3784                     "color": "red",
3785                     "value": 80
3786                   }
3787                 ]
3788               },
3789               "unit": "ms"
3790             },
3791             "overrides": []
3792           },
3793           "gridPos": {
3794             "h": 9,
3795             "w": 12,
3796             "x": 12,
3797             "y": 29
3798           },
3799           "id": 61,
3800           "options": {
3801             "legend": {
3802               "calcs": [
3803                 "mean",
3804                 "lastNotNull",
3805                 "max"
3806               ],
3807               "displayMode": "list",
3808               "placement": "bottom",
3809               "showLegend": true
3810             },
3811             "tooltip": {
3812               "mode": "single"
3813             }
3814           },
3815           "pluginVersion": "8.1.3",
3816           "targets": [
3817             {
3818               "datasource": {
3819                 "type": "prometheus",
3820                 "uid": "PBFA97CFB590B2093"
3821               },
3822               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3823               "hide": false,
3824               "legendFormat": "{{instance}} - {{quantile}}",
3825               "refId": "A"
3826             }
3827           ],
3828           "title": "Producer - LocalTimeMs",
3829           "type": "timeseries"
3830         },
3831         {
3832           "datasource": {
3833             "type": "prometheus",
3834             "uid": "PBFA97CFB590B2093"
3835           },
3836           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3837           "fieldConfig": {
3838             "defaults": {
3839               "color": {
3840                 "mode": "palette-classic"
3841               },
3842               "custom": {
3843                 "axisLabel": "",
3844                 "axisPlacement": "auto",
3845                 "barAlignment": 0,
3846                 "drawStyle": "line",
3847                 "fillOpacity": 10,
3848                 "gradientMode": "none",
3849                 "hideFrom": {
3850                   "legend": false,
3851                   "tooltip": false,
3852                   "viz": false
3853                 },
3854                 "lineInterpolation": "linear",
3855                 "lineWidth": 1,
3856                 "pointSize": 5,
3857                 "scaleDistribution": {
3858                   "type": "linear"
3859                 },
3860                 "showPoints": "never",
3861                 "spanNulls": true,
3862                 "stacking": {
3863                   "group": "A",
3864                   "mode": "none"
3865                 },
3866                 "thresholdsStyle": {
3867                   "mode": "off"
3868                 }
3869               },
3870               "links": [],
3871               "mappings": [],
3872               "thresholds": {
3873                 "mode": "absolute",
3874                 "steps": [
3875                   {
3876                     "color": "green"
3877                   },
3878                   {
3879                     "color": "red",
3880                     "value": 80
3881                   }
3882                 ]
3883               },
3884               "unit": "ms"
3885             },
3886             "overrides": []
3887           },
3888           "gridPos": {
3889             "h": 10,
3890             "w": 8,
3891             "x": 0,
3892             "y": 38
3893           },
3894           "id": 62,
3895           "options": {
3896             "legend": {
3897               "calcs": [
3898                 "mean",
3899                 "lastNotNull",
3900                 "max"
3901               ],
3902               "displayMode": "table",
3903               "placement": "bottom",
3904               "showLegend": true
3905             },
3906             "tooltip": {
3907               "mode": "single"
3908             }
3909           },
3910           "pluginVersion": "8.1.3",
3911           "targets": [
3912             {
3913               "datasource": {
3914                 "type": "prometheus",
3915                 "uid": "PBFA97CFB590B2093"
3916               },
3917               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3918               "hide": false,
3919               "legendFormat": "{{instance}} - {{quantile}}",
3920               "refId": "A"
3921             }
3922           ],
3923           "title": "Producer - RemoteTimeMs",
3924           "type": "timeseries"
3925         },
3926         {
3927           "datasource": {
3928             "type": "prometheus",
3929             "uid": "PBFA97CFB590B2093"
3930           },
3931           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3932           "fieldConfig": {
3933             "defaults": {
3934               "color": {
3935                 "mode": "palette-classic"
3936               },
3937               "custom": {
3938                 "axisLabel": "",
3939                 "axisPlacement": "auto",
3940                 "barAlignment": 0,
3941                 "drawStyle": "line",
3942                 "fillOpacity": 10,
3943                 "gradientMode": "none",
3944                 "hideFrom": {
3945                   "legend": false,
3946                   "tooltip": false,
3947                   "viz": false
3948                 },
3949                 "lineInterpolation": "linear",
3950                 "lineWidth": 1,
3951                 "pointSize": 5,
3952                 "scaleDistribution": {
3953                   "type": "linear"
3954                 },
3955                 "showPoints": "never",
3956                 "spanNulls": true,
3957                 "stacking": {
3958                   "group": "A",
3959                   "mode": "none"
3960                 },
3961                 "thresholdsStyle": {
3962                   "mode": "off"
3963                 }
3964               },
3965               "links": [],
3966               "mappings": [],
3967               "thresholds": {
3968                 "mode": "absolute",
3969                 "steps": [
3970                   {
3971                     "color": "green"
3972                   },
3973                   {
3974                     "color": "red",
3975                     "value": 80
3976                   }
3977                 ]
3978               },
3979               "unit": "ms"
3980             },
3981             "overrides": []
3982           },
3983           "gridPos": {
3984             "h": 10,
3985             "w": 8,
3986             "x": 8,
3987             "y": 38
3988           },
3989           "id": 63,
3990           "options": {
3991             "legend": {
3992               "calcs": [
3993                 "mean",
3994                 "lastNotNull",
3995                 "max"
3996               ],
3997               "displayMode": "table",
3998               "placement": "bottom",
3999               "showLegend": true
4000             },
4001             "tooltip": {
4002               "mode": "single"
4003             }
4004           },
4005           "pluginVersion": "8.1.3",
4006           "targets": [
4007             {
4008               "datasource": {
4009                 "type": "prometheus",
4010                 "uid": "PBFA97CFB590B2093"
4011               },
4012               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
4013               "hide": false,
4014               "legendFormat": "{{instance}} - {{quantile}}",
4015               "refId": "A"
4016             }
4017           ],
4018           "title": "Producer - ResponseQueueTimeMs",
4019           "type": "timeseries"
4020         },
4021         {
4022           "datasource": {
4023             "type": "prometheus",
4024             "uid": "PBFA97CFB590B2093"
4025           },
4026           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4027           "fieldConfig": {
4028             "defaults": {
4029               "color": {
4030                 "mode": "palette-classic"
4031               },
4032               "custom": {
4033                 "axisLabel": "",
4034                 "axisPlacement": "auto",
4035                 "barAlignment": 0,
4036                 "drawStyle": "line",
4037                 "fillOpacity": 10,
4038                 "gradientMode": "none",
4039                 "hideFrom": {
4040                   "legend": false,
4041                   "tooltip": false,
4042                   "viz": false
4043                 },
4044                 "lineInterpolation": "linear",
4045                 "lineWidth": 1,
4046                 "pointSize": 5,
4047                 "scaleDistribution": {
4048                   "type": "linear"
4049                 },
4050                 "showPoints": "never",
4051                 "spanNulls": true,
4052                 "stacking": {
4053                   "group": "A",
4054                   "mode": "none"
4055                 },
4056                 "thresholdsStyle": {
4057                   "mode": "off"
4058                 }
4059               },
4060               "links": [],
4061               "mappings": [],
4062               "min": 0,
4063               "thresholds": {
4064                 "mode": "absolute",
4065                 "steps": [
4066                   {
4067                     "color": "green"
4068                   },
4069                   {
4070                     "color": "red",
4071                     "value": 80
4072                   }
4073                 ]
4074               },
4075               "unit": "ms"
4076             },
4077             "overrides": []
4078           },
4079           "gridPos": {
4080             "h": 10,
4081             "w": 8,
4082             "x": 16,
4083             "y": 38
4084           },
4085           "id": 64,
4086           "options": {
4087             "legend": {
4088               "calcs": [
4089                 "mean",
4090                 "lastNotNull",
4091                 "max"
4092               ],
4093               "displayMode": "table",
4094               "placement": "bottom",
4095               "showLegend": true
4096             },
4097             "tooltip": {
4098               "mode": "single"
4099             }
4100           },
4101           "pluginVersion": "8.1.3",
4102           "targets": [
4103             {
4104               "datasource": {
4105                 "type": "prometheus",
4106                 "uid": "PBFA97CFB590B2093"
4107               },
4108               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
4109               "hide": false,
4110               "legendFormat": "{{instance}} - {{quantile}}",
4111               "refId": "A"
4112             }
4113           ],
4114           "title": "Producer - ResponseSendTimeMs",
4115           "type": "timeseries"
4116         }
4117       ],
4118       "targets": [
4119         {
4120           "datasource": {
4121             "type": "datasource",
4122             "uid": "grafana"
4123           },
4124           "refId": "A"
4125         }
4126       ],
4127       "title": "Producer Performance",
4128       "type": "row"
4129     },
4130     {
4131       "collapsed": true,
4132       "datasource": {
4133         "type": "datasource",
4134         "uid": "grafana"
4135       },
4136       "gridPos": {
4137         "h": 1,
4138         "w": 24,
4139         "x": 0,
4140         "y": 50
4141       },
4142       "id": 68,
4143       "panels": [
4144         {
4145           "datasource": {
4146             "type": "prometheus",
4147             "uid": "PBFA97CFB590B2093"
4148           },
4149           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4150           "fieldConfig": {
4151             "defaults": {
4152               "color": {
4153                 "mode": "palette-classic"
4154               },
4155               "custom": {
4156                 "axisLabel": "",
4157                 "axisPlacement": "auto",
4158                 "barAlignment": 0,
4159                 "drawStyle": "line",
4160                 "fillOpacity": 10,
4161                 "gradientMode": "none",
4162                 "hideFrom": {
4163                   "legend": false,
4164                   "tooltip": false,
4165                   "viz": false
4166                 },
4167                 "lineInterpolation": "linear",
4168                 "lineWidth": 1,
4169                 "pointSize": 5,
4170                 "scaleDistribution": {
4171                   "type": "linear"
4172                 },
4173                 "showPoints": "never",
4174                 "spanNulls": true,
4175                 "stacking": {
4176                   "group": "A",
4177                   "mode": "none"
4178                 },
4179                 "thresholdsStyle": {
4180                   "mode": "off"
4181                 }
4182               },
4183               "links": [],
4184               "mappings": [],
4185               "min": 0,
4186               "thresholds": {
4187                 "mode": "absolute",
4188                 "steps": [
4189                   {
4190                     "color": "green"
4191                   },
4192                   {
4193                     "color": "red",
4194                     "value": 80
4195                   }
4196                 ]
4197               },
4198               "unit": "ms"
4199             },
4200             "overrides": []
4201           },
4202           "gridPos": {
4203             "h": 9,
4204             "w": 12,
4205             "x": 0,
4206             "y": 30
4207           },
4208           "id": 69,
4209           "options": {
4210             "legend": {
4211               "calcs": [
4212                 "mean",
4213                 "lastNotNull",
4214                 "max"
4215               ],
4216               "displayMode": "list",
4217               "placement": "bottom",
4218               "showLegend": true
4219             },
4220             "tooltip": {
4221               "mode": "single"
4222             }
4223           },
4224           "pluginVersion": "8.1.3",
4225           "targets": [
4226             {
4227               "datasource": {
4228                 "type": "prometheus",
4229                 "uid": "PBFA97CFB590B2093"
4230               },
4231               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4232               "hide": false,
4233               "legendFormat": "{{instance}} - {{quantile}}",
4234               "refId": "A"
4235             }
4236           ],
4237           "title": "Consumer - RequestQueueTimeMs",
4238           "type": "timeseries"
4239         },
4240         {
4241           "datasource": {
4242             "type": "prometheus",
4243             "uid": "PBFA97CFB590B2093"
4244           },
4245           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4246           "fieldConfig": {
4247             "defaults": {
4248               "color": {
4249                 "mode": "palette-classic"
4250               },
4251               "custom": {
4252                 "axisLabel": "",
4253                 "axisPlacement": "auto",
4254                 "barAlignment": 0,
4255                 "drawStyle": "line",
4256                 "fillOpacity": 10,
4257                 "gradientMode": "none",
4258                 "hideFrom": {
4259                   "legend": false,
4260                   "tooltip": false,
4261                   "viz": false
4262                 },
4263                 "lineInterpolation": "linear",
4264                 "lineWidth": 1,
4265                 "pointSize": 5,
4266                 "scaleDistribution": {
4267                   "type": "linear"
4268                 },
4269                 "showPoints": "never",
4270                 "spanNulls": true,
4271                 "stacking": {
4272                   "group": "A",
4273                   "mode": "none"
4274                 },
4275                 "thresholdsStyle": {
4276                   "mode": "off"
4277                 }
4278               },
4279               "links": [],
4280               "mappings": [],
4281               "thresholds": {
4282                 "mode": "absolute",
4283                 "steps": [
4284                   {
4285                     "color": "green"
4286                   },
4287                   {
4288                     "color": "red",
4289                     "value": 80
4290                   }
4291                 ]
4292               },
4293               "unit": "ms"
4294             },
4295             "overrides": []
4296           },
4297           "gridPos": {
4298             "h": 9,
4299             "w": 12,
4300             "x": 12,
4301             "y": 30
4302           },
4303           "id": 70,
4304           "options": {
4305             "legend": {
4306               "calcs": [
4307                 "mean",
4308                 "lastNotNull",
4309                 "max"
4310               ],
4311               "displayMode": "list",
4312               "placement": "bottom",
4313               "showLegend": true
4314             },
4315             "tooltip": {
4316               "mode": "single"
4317             }
4318           },
4319           "pluginVersion": "8.1.3",
4320           "targets": [
4321             {
4322               "datasource": {
4323                 "type": "prometheus",
4324                 "uid": "PBFA97CFB590B2093"
4325               },
4326               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4327               "hide": false,
4328               "legendFormat": "{{instance}} - {{quantile}}",
4329               "refId": "A"
4330             }
4331           ],
4332           "title": "Consumer - LocalTimeMs",
4333           "type": "timeseries"
4334         },
4335         {
4336           "datasource": {
4337             "type": "prometheus",
4338             "uid": "PBFA97CFB590B2093"
4339           },
4340           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4341           "fieldConfig": {
4342             "defaults": {
4343               "color": {
4344                 "mode": "palette-classic"
4345               },
4346               "custom": {
4347                 "axisLabel": "",
4348                 "axisPlacement": "auto",
4349                 "barAlignment": 0,
4350                 "drawStyle": "line",
4351                 "fillOpacity": 10,
4352                 "gradientMode": "none",
4353                 "hideFrom": {
4354                   "legend": false,
4355                   "tooltip": false,
4356                   "viz": false
4357                 },
4358                 "lineInterpolation": "linear",
4359                 "lineWidth": 1,
4360                 "pointSize": 5,
4361                 "scaleDistribution": {
4362                   "type": "linear"
4363                 },
4364                 "showPoints": "never",
4365                 "spanNulls": true,
4366                 "stacking": {
4367                   "group": "A",
4368                   "mode": "none"
4369                 },
4370                 "thresholdsStyle": {
4371                   "mode": "off"
4372                 }
4373               },
4374               "links": [],
4375               "mappings": [],
4376               "thresholds": {
4377                 "mode": "absolute",
4378                 "steps": [
4379                   {
4380                     "color": "green"
4381                   },
4382                   {
4383                     "color": "red",
4384                     "value": 80
4385                   }
4386                 ]
4387               },
4388               "unit": "ms"
4389             },
4390             "overrides": []
4391           },
4392           "gridPos": {
4393             "h": 9,
4394             "w": 8,
4395             "x": 0,
4396             "y": 39
4397           },
4398           "id": 71,
4399           "options": {
4400             "legend": {
4401               "calcs": [
4402                 "mean",
4403                 "lastNotNull",
4404                 "max"
4405               ],
4406               "displayMode": "table",
4407               "placement": "bottom",
4408               "showLegend": true
4409             },
4410             "tooltip": {
4411               "mode": "single"
4412             }
4413           },
4414           "pluginVersion": "8.1.3",
4415           "targets": [
4416             {
4417               "datasource": {
4418                 "type": "prometheus",
4419                 "uid": "PBFA97CFB590B2093"
4420               },
4421               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4422               "hide": false,
4423               "legendFormat": "{{instance}} - {{quantile}}",
4424               "refId": "A"
4425             }
4426           ],
4427           "title": "Consumer - RemoteTimeMs",
4428           "type": "timeseries"
4429         },
4430         {
4431           "datasource": {
4432             "type": "prometheus",
4433             "uid": "PBFA97CFB590B2093"
4434           },
4435           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4436           "fieldConfig": {
4437             "defaults": {
4438               "color": {
4439                 "mode": "palette-classic"
4440               },
4441               "custom": {
4442                 "axisLabel": "",
4443                 "axisPlacement": "auto",
4444                 "barAlignment": 0,
4445                 "drawStyle": "line",
4446                 "fillOpacity": 10,
4447                 "gradientMode": "none",
4448                 "hideFrom": {
4449                   "legend": false,
4450                   "tooltip": false,
4451                   "viz": false
4452                 },
4453                 "lineInterpolation": "linear",
4454                 "lineWidth": 1,
4455                 "pointSize": 5,
4456                 "scaleDistribution": {
4457                   "type": "linear"
4458                 },
4459                 "showPoints": "never",
4460                 "spanNulls": true,
4461                 "stacking": {
4462                   "group": "A",
4463                   "mode": "none"
4464                 },
4465                 "thresholdsStyle": {
4466                   "mode": "off"
4467                 }
4468               },
4469               "links": [],
4470               "mappings": [],
4471               "min": 0,
4472               "thresholds": {
4473                 "mode": "absolute",
4474                 "steps": [
4475                   {
4476                     "color": "green"
4477                   },
4478                   {
4479                     "color": "red",
4480                     "value": 80
4481                   }
4482                 ]
4483               },
4484               "unit": "ms"
4485             },
4486             "overrides": []
4487           },
4488           "gridPos": {
4489             "h": 9,
4490             "w": 8,
4491             "x": 8,
4492             "y": 39
4493           },
4494           "id": 72,
4495           "options": {
4496             "legend": {
4497               "calcs": [
4498                 "mean",
4499                 "lastNotNull",
4500                 "max"
4501               ],
4502               "displayMode": "table",
4503               "placement": "bottom",
4504               "showLegend": true
4505             },
4506             "tooltip": {
4507               "mode": "single"
4508             }
4509           },
4510           "pluginVersion": "8.1.3",
4511           "targets": [
4512             {
4513               "datasource": {
4514                 "type": "prometheus",
4515                 "uid": "PBFA97CFB590B2093"
4516               },
4517               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4518               "hide": false,
4519               "legendFormat": "{{instance}} - {{quantile}}",
4520               "refId": "A"
4521             }
4522           ],
4523           "title": "Consumer - ResponseQueueTimeMs",
4524           "type": "timeseries"
4525         },
4526         {
4527           "datasource": {
4528             "type": "prometheus",
4529             "uid": "PBFA97CFB590B2093"
4530           },
4531           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4532           "fieldConfig": {
4533             "defaults": {
4534               "color": {
4535                 "mode": "palette-classic"
4536               },
4537               "custom": {
4538                 "axisLabel": "",
4539                 "axisPlacement": "auto",
4540                 "barAlignment": 0,
4541                 "drawStyle": "line",
4542                 "fillOpacity": 10,
4543                 "gradientMode": "none",
4544                 "hideFrom": {
4545                   "legend": false,
4546                   "tooltip": false,
4547                   "viz": false
4548                 },
4549                 "lineInterpolation": "linear",
4550                 "lineWidth": 1,
4551                 "pointSize": 5,
4552                 "scaleDistribution": {
4553                   "type": "linear"
4554                 },
4555                 "showPoints": "never",
4556                 "spanNulls": true,
4557                 "stacking": {
4558                   "group": "A",
4559                   "mode": "none"
4560                 },
4561                 "thresholdsStyle": {
4562                   "mode": "off"
4563                 }
4564               },
4565               "links": [],
4566               "mappings": [],
4567               "min": 0,
4568               "thresholds": {
4569                 "mode": "absolute",
4570                 "steps": [
4571                   {
4572                     "color": "green"
4573                   },
4574                   {
4575                     "color": "red",
4576                     "value": 80
4577                   }
4578                 ]
4579               },
4580               "unit": "ms"
4581             },
4582             "overrides": []
4583           },
4584           "gridPos": {
4585             "h": 9,
4586             "w": 8,
4587             "x": 16,
4588             "y": 39
4589           },
4590           "id": 73,
4591           "options": {
4592             "legend": {
4593               "calcs": [
4594                 "mean",
4595                 "lastNotNull",
4596                 "max"
4597               ],
4598               "displayMode": "table",
4599               "placement": "bottom",
4600               "showLegend": true
4601             },
4602             "tooltip": {
4603               "mode": "single"
4604             }
4605           },
4606           "pluginVersion": "8.1.3",
4607           "targets": [
4608             {
4609               "datasource": {
4610                 "type": "prometheus",
4611                 "uid": "PBFA97CFB590B2093"
4612               },
4613               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4614               "hide": false,
4615               "legendFormat": "{{instance}} - {{quantile}}",
4616               "refId": "A"
4617             }
4618           ],
4619           "title": "Consumer - ResponseSendTimeMs",
4620           "type": "timeseries"
4621         }
4622       ],
4623       "targets": [
4624         {
4625           "datasource": {
4626             "type": "datasource",
4627             "uid": "grafana"
4628           },
4629           "refId": "A"
4630         }
4631       ],
4632       "title": "Consumer Performance",
4633       "type": "row"
4634     },
4635     {
4636       "collapsed": true,
4637       "datasource": {
4638         "type": "datasource",
4639         "uid": "grafana"
4640       },
4641       "gridPos": {
4642         "h": 1,
4643         "w": 24,
4644         "x": 0,
4645         "y": 51
4646       },
4647       "id": 66,
4648       "panels": [
4649         {
4650           "datasource": {
4651             "type": "prometheus",
4652             "uid": "PBFA97CFB590B2093"
4653           },
4654           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4655           "fieldConfig": {
4656             "defaults": {
4657               "color": {
4658                 "mode": "palette-classic"
4659               },
4660               "custom": {
4661                 "axisLabel": "",
4662                 "axisPlacement": "auto",
4663                 "barAlignment": 0,
4664                 "drawStyle": "line",
4665                 "fillOpacity": 10,
4666                 "gradientMode": "none",
4667                 "hideFrom": {
4668                   "legend": false,
4669                   "tooltip": false,
4670                   "viz": false
4671                 },
4672                 "lineInterpolation": "linear",
4673                 "lineWidth": 1,
4674                 "pointSize": 5,
4675                 "scaleDistribution": {
4676                   "type": "linear"
4677                 },
4678                 "showPoints": "never",
4679                 "spanNulls": true,
4680                 "stacking": {
4681                   "group": "A",
4682                   "mode": "none"
4683                 },
4684                 "thresholdsStyle": {
4685                   "mode": "off"
4686                 }
4687               },
4688               "links": [],
4689               "mappings": [],
4690               "min": 0,
4691               "thresholds": {
4692                 "mode": "absolute",
4693                 "steps": [
4694                   {
4695                     "color": "green"
4696                   },
4697                   {
4698                     "color": "red",
4699                     "value": 80
4700                   }
4701                 ]
4702               },
4703               "unit": "ms"
4704             },
4705             "overrides": []
4706           },
4707           "gridPos": {
4708             "h": 9,
4709             "w": 12,
4710             "x": 0,
4711             "y": 31
4712           },
4713           "id": 74,
4714           "options": {
4715             "legend": {
4716               "calcs": [
4717                 "mean",
4718                 "lastNotNull",
4719                 "max"
4720               ],
4721               "displayMode": "list",
4722               "placement": "bottom",
4723               "showLegend": true
4724             },
4725             "tooltip": {
4726               "mode": "single"
4727             }
4728           },
4729           "pluginVersion": "8.1.3",
4730           "targets": [
4731             {
4732               "datasource": {
4733                 "type": "prometheus",
4734                 "uid": "PBFA97CFB590B2093"
4735               },
4736               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4737               "hide": false,
4738               "legendFormat": "{{instance}} - {{quantile}}",
4739               "refId": "A"
4740             }
4741           ],
4742           "title": "FetchFollower - RequestQueueTimeMs",
4743           "type": "timeseries"
4744         },
4745         {
4746           "datasource": {
4747             "type": "prometheus",
4748             "uid": "PBFA97CFB590B2093"
4749           },
4750           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4751           "fieldConfig": {
4752             "defaults": {
4753               "color": {
4754                 "mode": "palette-classic"
4755               },
4756               "custom": {
4757                 "axisLabel": "",
4758                 "axisPlacement": "auto",
4759                 "barAlignment": 0,
4760                 "drawStyle": "line",
4761                 "fillOpacity": 10,
4762                 "gradientMode": "none",
4763                 "hideFrom": {
4764                   "legend": false,
4765                   "tooltip": false,
4766                   "viz": false
4767                 },
4768                 "lineInterpolation": "linear",
4769                 "lineWidth": 1,
4770                 "pointSize": 5,
4771                 "scaleDistribution": {
4772                   "type": "linear"
4773                 },
4774                 "showPoints": "never",
4775                 "spanNulls": true,
4776                 "stacking": {
4777                   "group": "A",
4778                   "mode": "none"
4779                 },
4780                 "thresholdsStyle": {
4781                   "mode": "off"
4782                 }
4783               },
4784               "links": [],
4785               "mappings": [],
4786               "min": 0,
4787               "thresholds": {
4788                 "mode": "absolute",
4789                 "steps": [
4790                   {
4791                     "color": "green"
4792                   },
4793                   {
4794                     "color": "red",
4795                     "value": 80
4796                   }
4797                 ]
4798               },
4799               "unit": "ms"
4800             },
4801             "overrides": []
4802           },
4803           "gridPos": {
4804             "h": 9,
4805             "w": 12,
4806             "x": 12,
4807             "y": 31
4808           },
4809           "id": 75,
4810           "options": {
4811             "legend": {
4812               "calcs": [
4813                 "mean",
4814                 "lastNotNull",
4815                 "max"
4816               ],
4817               "displayMode": "list",
4818               "placement": "bottom",
4819               "showLegend": true
4820             },
4821             "tooltip": {
4822               "mode": "single"
4823             }
4824           },
4825           "pluginVersion": "8.1.3",
4826           "targets": [
4827             {
4828               "datasource": {
4829                 "type": "prometheus",
4830                 "uid": "PBFA97CFB590B2093"
4831               },
4832               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4833               "hide": false,
4834               "legendFormat": "{{instance}} - {{quantile}}",
4835               "refId": "A"
4836             }
4837           ],
4838           "title": "FetchFollower - LocalTimeMs",
4839           "type": "timeseries"
4840         },
4841         {
4842           "datasource": {
4843             "type": "prometheus",
4844             "uid": "PBFA97CFB590B2093"
4845           },
4846           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4847           "fieldConfig": {
4848             "defaults": {
4849               "color": {
4850                 "mode": "palette-classic"
4851               },
4852               "custom": {
4853                 "axisLabel": "",
4854                 "axisPlacement": "auto",
4855                 "barAlignment": 0,
4856                 "drawStyle": "line",
4857                 "fillOpacity": 10,
4858                 "gradientMode": "none",
4859                 "hideFrom": {
4860                   "legend": false,
4861                   "tooltip": false,
4862                   "viz": false
4863                 },
4864                 "lineInterpolation": "linear",
4865                 "lineWidth": 1,
4866                 "pointSize": 5,
4867                 "scaleDistribution": {
4868                   "type": "linear"
4869                 },
4870                 "showPoints": "never",
4871                 "spanNulls": true,
4872                 "stacking": {
4873                   "group": "A",
4874                   "mode": "none"
4875                 },
4876                 "thresholdsStyle": {
4877                   "mode": "off"
4878                 }
4879               },
4880               "links": [],
4881               "mappings": [],
4882               "thresholds": {
4883                 "mode": "absolute",
4884                 "steps": [
4885                   {
4886                     "color": "green"
4887                   },
4888                   {
4889                     "color": "red",
4890                     "value": 80
4891                   }
4892                 ]
4893               },
4894               "unit": "ms"
4895             },
4896             "overrides": []
4897           },
4898           "gridPos": {
4899             "h": 9,
4900             "w": 8,
4901             "x": 0,
4902             "y": 40
4903           },
4904           "id": 76,
4905           "options": {
4906             "legend": {
4907               "calcs": [
4908                 "mean",
4909                 "lastNotNull",
4910                 "max"
4911               ],
4912               "displayMode": "table",
4913               "placement": "bottom",
4914               "showLegend": true
4915             },
4916             "tooltip": {
4917               "mode": "single"
4918             }
4919           },
4920           "pluginVersion": "8.1.3",
4921           "targets": [
4922             {
4923               "datasource": {
4924                 "type": "prometheus",
4925                 "uid": "PBFA97CFB590B2093"
4926               },
4927               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4928               "hide": false,
4929               "legendFormat": "{{instance}} - {{quantile}}",
4930               "refId": "A"
4931             }
4932           ],
4933           "title": "FetchFollower - RemoteTimeMs",
4934           "type": "timeseries"
4935         },
4936         {
4937           "datasource": {
4938             "type": "prometheus",
4939             "uid": "PBFA97CFB590B2093"
4940           },
4941           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4942           "fieldConfig": {
4943             "defaults": {
4944               "color": {
4945                 "mode": "palette-classic"
4946               },
4947               "custom": {
4948                 "axisLabel": "",
4949                 "axisPlacement": "auto",
4950                 "barAlignment": 0,
4951                 "drawStyle": "line",
4952                 "fillOpacity": 10,
4953                 "gradientMode": "none",
4954                 "hideFrom": {
4955                   "legend": false,
4956                   "tooltip": false,
4957                   "viz": false
4958                 },
4959                 "lineInterpolation": "linear",
4960                 "lineWidth": 1,
4961                 "pointSize": 5,
4962                 "scaleDistribution": {
4963                   "type": "linear"
4964                 },
4965                 "showPoints": "never",
4966                 "spanNulls": true,
4967                 "stacking": {
4968                   "group": "A",
4969                   "mode": "none"
4970                 },
4971                 "thresholdsStyle": {
4972                   "mode": "off"
4973                 }
4974               },
4975               "links": [],
4976               "mappings": [],
4977               "thresholds": {
4978                 "mode": "absolute",
4979                 "steps": [
4980                   {
4981                     "color": "green"
4982                   },
4983                   {
4984                     "color": "red",
4985                     "value": 80
4986                   }
4987                 ]
4988               },
4989               "unit": "ms"
4990             },
4991             "overrides": []
4992           },
4993           "gridPos": {
4994             "h": 9,
4995             "w": 8,
4996             "x": 8,
4997             "y": 40
4998           },
4999           "id": 77,
5000           "options": {
5001             "legend": {
5002               "calcs": [
5003                 "mean",
5004                 "lastNotNull",
5005                 "max"
5006               ],
5007               "displayMode": "table",
5008               "placement": "bottom",
5009               "showLegend": true
5010             },
5011             "tooltip": {
5012               "mode": "single"
5013             }
5014           },
5015           "pluginVersion": "8.1.3",
5016           "targets": [
5017             {
5018               "datasource": {
5019                 "type": "prometheus",
5020                 "uid": "PBFA97CFB590B2093"
5021               },
5022               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
5023               "hide": false,
5024               "legendFormat": "{{instance}} - {{quantile}}",
5025               "refId": "A"
5026             }
5027           ],
5028           "title": "FetchFollower - ResponseQueueTimeMs",
5029           "type": "timeseries"
5030         },
5031         {
5032           "datasource": {
5033             "type": "prometheus",
5034             "uid": "PBFA97CFB590B2093"
5035           },
5036           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
5037           "fieldConfig": {
5038             "defaults": {
5039               "color": {
5040                 "mode": "palette-classic"
5041               },
5042               "custom": {
5043                 "axisLabel": "",
5044                 "axisPlacement": "auto",
5045                 "barAlignment": 0,
5046                 "drawStyle": "line",
5047                 "fillOpacity": 10,
5048                 "gradientMode": "none",
5049                 "hideFrom": {
5050                   "legend": false,
5051                   "tooltip": false,
5052                   "viz": false
5053                 },
5054                 "lineInterpolation": "linear",
5055                 "lineWidth": 1,
5056                 "pointSize": 5,
5057                 "scaleDistribution": {
5058                   "type": "linear"
5059                 },
5060                 "showPoints": "never",
5061                 "spanNulls": true,
5062                 "stacking": {
5063                   "group": "A",
5064                   "mode": "none"
5065                 },
5066                 "thresholdsStyle": {
5067                   "mode": "off"
5068                 }
5069               },
5070               "links": [],
5071               "mappings": [],
5072               "min": 0,
5073               "thresholds": {
5074                 "mode": "absolute",
5075                 "steps": [
5076                   {
5077                     "color": "green"
5078                   },
5079                   {
5080                     "color": "red",
5081                     "value": 80
5082                   }
5083                 ]
5084               },
5085               "unit": "ms"
5086             },
5087             "overrides": []
5088           },
5089           "gridPos": {
5090             "h": 9,
5091             "w": 8,
5092             "x": 16,
5093             "y": 40
5094           },
5095           "id": 78,
5096           "options": {
5097             "legend": {
5098               "calcs": [
5099                 "mean",
5100                 "lastNotNull",
5101                 "max"
5102               ],
5103               "displayMode": "table",
5104               "placement": "bottom",
5105               "showLegend": true
5106             },
5107             "tooltip": {
5108               "mode": "single"
5109             }
5110           },
5111           "pluginVersion": "8.1.3",
5112           "targets": [
5113             {
5114               "datasource": {
5115                 "type": "prometheus",
5116                 "uid": "PBFA97CFB590B2093"
5117               },
5118               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
5119               "hide": false,
5120               "legendFormat": "{{instance}} - {{quantile}}",
5121               "refId": "A"
5122             }
5123           ],
5124           "title": "FetchFollower - ResponseSendTimeMs",
5125           "type": "timeseries"
5126         }
5127       ],
5128       "targets": [
5129         {
5130           "datasource": {
5131             "type": "datasource",
5132             "uid": "grafana"
5133           },
5134           "refId": "A"
5135         }
5136       ],
5137       "title": "Fetch Follower Performance",
5138       "type": "row"
5139     },
5140     {
5141       "collapsed": true,
5142       "datasource": {
5143         "type": "datasource",
5144         "uid": "grafana"
5145       },
5146       "gridPos": {
5147         "h": 1,
5148         "w": 24,
5149         "x": 0,
5150         "y": 52
5151       },
5152       "id": 102,
5153       "panels": [
5154         {
5155           "datasource": {
5156             "type": "prometheus",
5157             "uid": "PBFA97CFB590B2093"
5158           },
5159           "fieldConfig": {
5160             "defaults": {
5161               "color": {
5162                 "mode": "palette-classic"
5163               },
5164               "custom": {
5165                 "axisLabel": "",
5166                 "axisPlacement": "auto",
5167                 "barAlignment": 0,
5168                 "drawStyle": "line",
5169                 "fillOpacity": 10,
5170                 "gradientMode": "none",
5171                 "hideFrom": {
5172                   "legend": false,
5173                   "tooltip": false,
5174                   "viz": false
5175                 },
5176                 "lineInterpolation": "linear",
5177                 "lineWidth": 1,
5178                 "pointSize": 5,
5179                 "scaleDistribution": {
5180                   "type": "linear"
5181                 },
5182                 "showPoints": "never",
5183                 "spanNulls": true,
5184                 "stacking": {
5185                   "group": "A",
5186                   "mode": "none"
5187                 },
5188                 "thresholdsStyle": {
5189                   "mode": "off"
5190                 }
5191               },
5192               "links": [],
5193               "mappings": [],
5194               "thresholds": {
5195                 "mode": "absolute",
5196                 "steps": [
5197                   {
5198                     "color": "green"
5199                   },
5200                   {
5201                     "color": "red",
5202                     "value": 80
5203                   }
5204                 ]
5205               },
5206               "unit": "short"
5207             },
5208             "overrides": []
5209           },
5210           "gridPos": {
5211             "h": 8,
5212             "w": 12,
5213             "x": 0,
5214             "y": 32
5215           },
5216           "id": 98,
5217           "options": {
5218             "legend": {
5219               "calcs": [
5220                 "mean",
5221                 "lastNotNull",
5222                 "max"
5223               ],
5224               "displayMode": "table",
5225               "placement": "bottom",
5226               "showLegend": true
5227             },
5228             "tooltip": {
5229               "mode": "single"
5230             }
5231           },
5232           "pluginVersion": "8.1.3",
5233           "targets": [
5234             {
5235               "datasource": {
5236                 "type": "prometheus",
5237                 "uid": "PBFA97CFB590B2093"
5238               },
5239               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5240               "interval": "",
5241               "legendFormat": "{{listener}}",
5242               "refId": "A"
5243             }
5244           ],
5245           "title": "Connections count per listener",
5246           "type": "timeseries"
5247         },
5248         {
5249           "datasource": {
5250             "type": "prometheus",
5251             "uid": "PBFA97CFB590B2093"
5252           },
5253           "fieldConfig": {
5254             "defaults": {
5255               "color": {
5256                 "mode": "palette-classic"
5257               },
5258               "custom": {
5259                 "axisLabel": "",
5260                 "axisPlacement": "auto",
5261                 "barAlignment": 0,
5262                 "drawStyle": "line",
5263                 "fillOpacity": 10,
5264                 "gradientMode": "none",
5265                 "hideFrom": {
5266                   "legend": false,
5267                   "tooltip": false,
5268                   "viz": false
5269                 },
5270                 "lineInterpolation": "linear",
5271                 "lineWidth": 1,
5272                 "pointSize": 5,
5273                 "scaleDistribution": {
5274                   "type": "linear"
5275                 },
5276                 "showPoints": "never",
5277                 "spanNulls": true,
5278                 "stacking": {
5279                   "group": "A",
5280                   "mode": "none"
5281                 },
5282                 "thresholdsStyle": {
5283                   "mode": "off"
5284                 }
5285               },
5286               "links": [],
5287               "mappings": [],
5288               "thresholds": {
5289                 "mode": "absolute",
5290                 "steps": [
5291                   {
5292                     "color": "green"
5293                   },
5294                   {
5295                     "color": "red",
5296                     "value": 80
5297                   }
5298                 ]
5299               },
5300               "unit": "short"
5301             },
5302             "overrides": []
5303           },
5304           "gridPos": {
5305             "h": 8,
5306             "w": 12,
5307             "x": 12,
5308             "y": 32
5309           },
5310           "id": 100,
5311           "options": {
5312             "legend": {
5313               "calcs": [
5314                 "mean",
5315                 "lastNotNull",
5316                 "max"
5317               ],
5318               "displayMode": "table",
5319               "placement": "bottom",
5320               "showLegend": true
5321             },
5322             "tooltip": {
5323               "mode": "single"
5324             }
5325           },
5326           "pluginVersion": "8.1.3",
5327           "targets": [
5328             {
5329               "datasource": {
5330                 "type": "prometheus",
5331                 "uid": "PBFA97CFB590B2093"
5332               },
5333               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5334               "interval": "",
5335               "legendFormat": "{{instance}}",
5336               "refId": "A"
5337             }
5338           ],
5339           "title": "Connections count per broker",
5340           "type": "timeseries"
5341         },
5342         {
5343           "datasource": {
5344             "type": "prometheus",
5345             "uid": "PBFA97CFB590B2093"
5346           },
5347           "fieldConfig": {
5348             "defaults": {
5349               "color": {
5350                 "mode": "palette-classic"
5351               },
5352               "custom": {
5353                 "axisLabel": "",
5354                 "axisPlacement": "auto",
5355                 "barAlignment": 0,
5356                 "drawStyle": "line",
5357                 "fillOpacity": 10,
5358                 "gradientMode": "none",
5359                 "hideFrom": {
5360                   "legend": false,
5361                   "tooltip": false,
5362                   "viz": false
5363                 },
5364                 "lineInterpolation": "linear",
5365                 "lineWidth": 1,
5366                 "pointSize": 5,
5367                 "scaleDistribution": {
5368                   "type": "linear"
5369                 },
5370                 "showPoints": "never",
5371                 "spanNulls": true,
5372                 "stacking": {
5373                   "group": "A",
5374                   "mode": "none"
5375                 },
5376                 "thresholdsStyle": {
5377                   "mode": "off"
5378                 }
5379               },
5380               "links": [],
5381               "mappings": [],
5382               "thresholds": {
5383                 "mode": "absolute",
5384                 "steps": [
5385                   {
5386                     "color": "green"
5387                   },
5388                   {
5389                     "color": "red",
5390                     "value": 80
5391                   }
5392                 ]
5393               },
5394               "unit": "short"
5395             },
5396             "overrides": []
5397           },
5398           "gridPos": {
5399             "h": 8,
5400             "w": 12,
5401             "x": 0,
5402             "y": 40
5403           },
5404           "id": 104,
5405           "options": {
5406             "legend": {
5407               "calcs": [
5408                 "mean",
5409                 "lastNotNull",
5410                 "max"
5411               ],
5412               "displayMode": "table",
5413               "placement": "bottom",
5414               "showLegend": true
5415             },
5416             "tooltip": {
5417               "mode": "single"
5418             }
5419           },
5420           "pluginVersion": "8.1.3",
5421           "targets": [
5422             {
5423               "datasource": {
5424                 "type": "prometheus",
5425                 "uid": "PBFA97CFB590B2093"
5426               },
5427               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5428               "interval": "",
5429               "legendFormat": "{{listener}}",
5430               "refId": "A"
5431             }
5432           ],
5433           "title": "Connections creation rate per listener",
5434           "type": "timeseries"
5435         },
5436         {
5437           "datasource": {
5438             "type": "prometheus",
5439             "uid": "PBFA97CFB590B2093"
5440           },
5441           "fieldConfig": {
5442             "defaults": {
5443               "color": {
5444                 "mode": "palette-classic"
5445               },
5446               "custom": {
5447                 "axisLabel": "",
5448                 "axisPlacement": "auto",
5449                 "barAlignment": 0,
5450                 "drawStyle": "line",
5451                 "fillOpacity": 10,
5452                 "gradientMode": "none",
5453                 "hideFrom": {
5454                   "legend": false,
5455                   "tooltip": false,
5456                   "viz": false
5457                 },
5458                 "lineInterpolation": "linear",
5459                 "lineWidth": 1,
5460                 "pointSize": 5,
5461                 "scaleDistribution": {
5462                   "type": "linear"
5463                 },
5464                 "showPoints": "never",
5465                 "spanNulls": true,
5466                 "stacking": {
5467                   "group": "A",
5468                   "mode": "none"
5469                 },
5470                 "thresholdsStyle": {
5471                   "mode": "off"
5472                 }
5473               },
5474               "links": [],
5475               "mappings": [],
5476               "thresholds": {
5477                 "mode": "absolute",
5478                 "steps": [
5479                   {
5480                     "color": "green"
5481                   },
5482                   {
5483                     "color": "red",
5484                     "value": 80
5485                   }
5486                 ]
5487               },
5488               "unit": "short"
5489             },
5490             "overrides": []
5491           },
5492           "gridPos": {
5493             "h": 8,
5494             "w": 12,
5495             "x": 12,
5496             "y": 40
5497           },
5498           "id": 106,
5499           "options": {
5500             "legend": {
5501               "calcs": [
5502                 "mean",
5503                 "lastNotNull",
5504                 "max"
5505               ],
5506               "displayMode": "table",
5507               "placement": "bottom",
5508               "showLegend": true
5509             },
5510             "tooltip": {
5511               "mode": "single"
5512             }
5513           },
5514           "pluginVersion": "8.1.3",
5515           "targets": [
5516             {
5517               "datasource": {
5518                 "type": "prometheus",
5519                 "uid": "PBFA97CFB590B2093"
5520               },
5521               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5522               "interval": "",
5523               "legendFormat": "{{instance}}",
5524               "refId": "A"
5525             }
5526           ],
5527           "title": "Connections creation rate per instance",
5528           "type": "timeseries"
5529         },
5530         {
5531           "datasource": {
5532             "type": "prometheus",
5533             "uid": "PBFA97CFB590B2093"
5534           },
5535           "fieldConfig": {
5536             "defaults": {
5537               "color": {
5538                 "mode": "palette-classic"
5539               },
5540               "custom": {
5541                 "axisLabel": "",
5542                 "axisPlacement": "auto",
5543                 "barAlignment": 0,
5544                 "drawStyle": "line",
5545                 "fillOpacity": 10,
5546                 "gradientMode": "none",
5547                 "hideFrom": {
5548                   "legend": false,
5549                   "tooltip": false,
5550                   "viz": false
5551                 },
5552                 "lineInterpolation": "linear",
5553                 "lineWidth": 1,
5554                 "pointSize": 5,
5555                 "scaleDistribution": {
5556                   "type": "linear"
5557                 },
5558                 "showPoints": "never",
5559                 "spanNulls": true,
5560                 "stacking": {
5561                   "group": "A",
5562                   "mode": "none"
5563                 },
5564                 "thresholdsStyle": {
5565                   "mode": "off"
5566                 }
5567               },
5568               "links": [],
5569               "mappings": [],
5570               "thresholds": {
5571                 "mode": "absolute",
5572                 "steps": [
5573                   {
5574                     "color": "green"
5575                   },
5576                   {
5577                     "color": "red",
5578                     "value": 80
5579                   }
5580                 ]
5581               },
5582               "unit": "short"
5583             },
5584             "overrides": []
5585           },
5586           "gridPos": {
5587             "h": 8,
5588             "w": 12,
5589             "x": 0,
5590             "y": 48
5591           },
5592           "id": 108,
5593           "options": {
5594             "legend": {
5595               "calcs": [
5596                 "mean",
5597                 "lastNotNull",
5598                 "max"
5599               ],
5600               "displayMode": "table",
5601               "placement": "bottom",
5602               "showLegend": true
5603             },
5604             "tooltip": {
5605               "mode": "single"
5606             }
5607           },
5608           "pluginVersion": "8.1.3",
5609           "targets": [
5610             {
5611               "datasource": {
5612                 "type": "prometheus",
5613                 "uid": "PBFA97CFB590B2093"
5614               },
5615               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5616               "interval": "",
5617               "legendFormat": "{{listener}}",
5618               "refId": "A"
5619             }
5620           ],
5621           "title": "Connections close rate per listener",
5622           "type": "timeseries"
5623         },
5624         {
5625           "datasource": {
5626             "type": "prometheus",
5627             "uid": "PBFA97CFB590B2093"
5628           },
5629           "fieldConfig": {
5630             "defaults": {
5631               "color": {
5632                 "mode": "palette-classic"
5633               },
5634               "custom": {
5635                 "axisLabel": "",
5636                 "axisPlacement": "auto",
5637                 "barAlignment": 0,
5638                 "drawStyle": "line",
5639                 "fillOpacity": 10,
5640                 "gradientMode": "none",
5641                 "hideFrom": {
5642                   "legend": false,
5643                   "tooltip": false,
5644                   "viz": false
5645                 },
5646                 "lineInterpolation": "linear",
5647                 "lineWidth": 1,
5648                 "pointSize": 5,
5649                 "scaleDistribution": {
5650                   "type": "linear"
5651                 },
5652                 "showPoints": "never",
5653                 "spanNulls": true,
5654                 "stacking": {
5655                   "group": "A",
5656                   "mode": "none"
5657                 },
5658                 "thresholdsStyle": {
5659                   "mode": "off"
5660                 }
5661               },
5662               "links": [],
5663               "mappings": [],
5664               "thresholds": {
5665                 "mode": "absolute",
5666                 "steps": [
5667                   {
5668                     "color": "green"
5669                   },
5670                   {
5671                     "color": "red",
5672                     "value": 80
5673                   }
5674                 ]
5675               },
5676               "unit": "short"
5677             },
5678             "overrides": []
5679           },
5680           "gridPos": {
5681             "h": 8,
5682             "w": 12,
5683             "x": 12,
5684             "y": 48
5685           },
5686           "id": 110,
5687           "options": {
5688             "legend": {
5689               "calcs": [
5690                 "mean",
5691                 "lastNotNull",
5692                 "max"
5693               ],
5694               "displayMode": "table",
5695               "placement": "bottom",
5696               "showLegend": true
5697             },
5698             "tooltip": {
5699               "mode": "single"
5700             }
5701           },
5702           "pluginVersion": "8.1.3",
5703           "targets": [
5704             {
5705               "datasource": {
5706                 "type": "prometheus",
5707                 "uid": "PBFA97CFB590B2093"
5708               },
5709               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5710               "interval": "",
5711               "legendFormat": "{{instance}}",
5712               "refId": "A"
5713             }
5714           ],
5715           "title": "Connections close rate per instance",
5716           "type": "timeseries"
5717         },
5718         {
5719           "datasource": {
5720             "type": "prometheus",
5721             "uid": "PBFA97CFB590B2093"
5722           },
5723           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5724           "fieldConfig": {
5725             "defaults": {
5726               "color": {
5727                 "mode": "palette-classic"
5728               },
5729               "custom": {
5730                 "axisLabel": "",
5731                 "axisPlacement": "auto",
5732                 "barAlignment": 0,
5733                 "drawStyle": "line",
5734                 "fillOpacity": 10,
5735                 "gradientMode": "none",
5736                 "hideFrom": {
5737                   "legend": false,
5738                   "tooltip": false,
5739                   "viz": false
5740                 },
5741                 "lineInterpolation": "linear",
5742                 "lineWidth": 1,
5743                 "pointSize": 5,
5744                 "scaleDistribution": {
5745                   "type": "linear"
5746                 },
5747                 "showPoints": "never",
5748                 "spanNulls": true,
5749                 "stacking": {
5750                   "group": "A",
5751                   "mode": "none"
5752                 },
5753                 "thresholdsStyle": {
5754                   "mode": "off"
5755                 }
5756               },
5757               "links": [],
5758               "mappings": [],
5759               "min": 0,
5760               "thresholds": {
5761                 "mode": "absolute",
5762                 "steps": [
5763                   {
5764                     "color": "green"
5765                   },
5766                   {
5767                     "color": "red",
5768                     "value": 80
5769                   }
5770                 ]
5771               },
5772               "unit": "percent"
5773             },
5774             "overrides": []
5775           },
5776           "gridPos": {
5777             "h": 8,
5778             "w": 12,
5779             "x": 0,
5780             "y": 56
5781           },
5782           "id": 112,
5783           "options": {
5784             "legend": {
5785               "calcs": [
5786                 "mean",
5787                 "lastNotNull",
5788                 "max"
5789               ],
5790               "displayMode": "table",
5791               "placement": "bottom",
5792               "showLegend": true
5793             },
5794             "tooltip": {
5795               "mode": "single"
5796             }
5797           },
5798           "pluginVersion": "8.1.3",
5799           "targets": [
5800             {
5801               "datasource": {
5802                 "type": "prometheus",
5803                 "uid": "PBFA97CFB590B2093"
5804               },
5805               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5806               "interval": "",
5807               "legendFormat": "{{instance}} - {{listener}}",
5808               "refId": "A"
5809             }
5810           ],
5811           "title": "Acceptor Blocked Percentage",
5812           "type": "timeseries"
5813         },
5814         {
5815           "datasource": {
5816             "type": "prometheus",
5817             "uid": "PBFA97CFB590B2093"
5818           },
5819           "fieldConfig": {
5820             "defaults": {
5821               "color": {
5822                 "mode": "palette-classic"
5823               },
5824               "custom": {
5825                 "axisLabel": "",
5826                 "axisPlacement": "auto",
5827                 "barAlignment": 0,
5828                 "drawStyle": "line",
5829                 "fillOpacity": 10,
5830                 "gradientMode": "none",
5831                 "hideFrom": {
5832                   "legend": false,
5833                   "tooltip": false,
5834                   "viz": false
5835                 },
5836                 "lineInterpolation": "linear",
5837                 "lineWidth": 1,
5838                 "pointSize": 5,
5839                 "scaleDistribution": {
5840                   "type": "linear"
5841                 },
5842                 "showPoints": "never",
5843                 "spanNulls": true,
5844                 "stacking": {
5845                   "group": "A",
5846                   "mode": "none"
5847                 },
5848                 "thresholdsStyle": {
5849                   "mode": "off"
5850                 }
5851               },
5852               "links": [],
5853               "mappings": [],
5854               "thresholds": {
5855                 "mode": "absolute",
5856                 "steps": [
5857                   {
5858                     "color": "green"
5859                   },
5860                   {
5861                     "color": "red",
5862                     "value": 80
5863                   }
5864                 ]
5865               },
5866               "unit": "short"
5867             },
5868             "overrides": []
5869           },
5870           "gridPos": {
5871             "h": 8,
5872             "w": 12,
5873             "x": 12,
5874             "y": 56
5875           },
5876           "id": 114,
5877           "options": {
5878             "legend": {
5879               "calcs": [
5880                 "mean",
5881                 "lastNotNull",
5882                 "max"
5883               ],
5884               "displayMode": "table",
5885               "placement": "bottom",
5886               "showLegend": true
5887             },
5888             "tooltip": {
5889               "mode": "single"
5890             }
5891           },
5892           "pluginVersion": "8.1.3",
5893           "targets": [
5894             {
5895               "datasource": {
5896                 "type": "prometheus",
5897                 "uid": "PBFA97CFB590B2093"
5898               },
5899               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5900               "interval": "",
5901               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5902               "refId": "A"
5903             }
5904           ],
5905           "title": "Connections per client version",
5906           "type": "timeseries"
5907         }
5908       ],
5909       "targets": [
5910         {
5911           "datasource": {
5912             "type": "datasource",
5913             "uid": "grafana"
5914           },
5915           "refId": "A"
5916         }
5917       ],
5918       "title": "Connections",
5919       "type": "row"
5920     },
5921     {
5922       "collapsed": true,
5923       "datasource": {
5924         "type": "datasource",
5925         "uid": "grafana"
5926       },
5927       "gridPos": {
5928         "h": 1,
5929         "w": 24,
5930         "x": 0,
5931         "y": 53
5932       },
5933       "id": 120,
5934       "panels": [
5935         {
5936           "datasource": {
5937             "type": "prometheus",
5938             "uid": "PBFA97CFB590B2093"
5939           },
5940           "description": "Number of consumer groups per group coordinator",
5941           "fieldConfig": {
5942             "defaults": {
5943               "color": {
5944                 "mode": "palette-classic"
5945               },
5946               "custom": {
5947                 "axisLabel": "",
5948                 "axisPlacement": "auto",
5949                 "barAlignment": 0,
5950                 "drawStyle": "line",
5951                 "fillOpacity": 10,
5952                 "gradientMode": "none",
5953                 "hideFrom": {
5954                   "legend": false,
5955                   "tooltip": false,
5956                   "viz": false
5957                 },
5958                 "lineInterpolation": "linear",
5959                 "lineWidth": 1,
5960                 "pointSize": 5,
5961                 "scaleDistribution": {
5962                   "type": "linear"
5963                 },
5964                 "showPoints": "never",
5965                 "spanNulls": true,
5966                 "stacking": {
5967                   "group": "A",
5968                   "mode": "none"
5969                 },
5970                 "thresholdsStyle": {
5971                   "mode": "off"
5972                 }
5973               },
5974               "links": [],
5975               "mappings": [],
5976               "thresholds": {
5977                 "mode": "absolute",
5978                 "steps": [
5979                   {
5980                     "color": "green"
5981                   },
5982                   {
5983                     "color": "red",
5984                     "value": 80
5985                   }
5986                 ]
5987               },
5988               "unit": "short"
5989             },
5990             "overrides": []
5991           },
5992           "gridPos": {
5993             "h": 8,
5994             "w": 12,
5995             "x": 0,
5996             "y": 33
5997           },
5998           "id": 116,
5999           "options": {
6000             "legend": {
6001               "calcs": [
6002                 "mean",
6003                 "lastNotNull",
6004                 "max"
6005               ],
6006               "displayMode": "table",
6007               "placement": "bottom",
6008               "showLegend": true
6009             },
6010             "tooltip": {
6011               "mode": "single"
6012             }
6013           },
6014           "pluginVersion": "8.1.3",
6015           "targets": [
6016             {
6017               "datasource": {
6018                 "type": "prometheus",
6019                 "uid": "PBFA97CFB590B2093"
6020               },
6021               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
6022               "instant": false,
6023               "interval": "",
6024               "legendFormat": "{{instance}}",
6025               "refId": "A"
6026             }
6027           ],
6028           "title": "Consumer groups number per coordinator",
6029           "type": "timeseries"
6030         },
6031         {
6032           "datasource": {
6033             "type": "prometheus",
6034             "uid": "PBFA97CFB590B2093"
6035           },
6036           "description": "Number of consumer group per state",
6037           "fieldConfig": {
6038             "defaults": {
6039               "color": {
6040                 "mode": "palette-classic"
6041               },
6042               "custom": {
6043                 "axisLabel": "",
6044                 "axisPlacement": "auto",
6045                 "barAlignment": 0,
6046                 "drawStyle": "line",
6047                 "fillOpacity": 10,
6048                 "gradientMode": "none",
6049                 "hideFrom": {
6050                   "legend": false,
6051                   "tooltip": false,
6052                   "viz": false
6053                 },
6054                 "lineInterpolation": "linear",
6055                 "lineWidth": 1,
6056                 "pointSize": 5,
6057                 "scaleDistribution": {
6058                   "type": "linear"
6059                 },
6060                 "showPoints": "never",
6061                 "spanNulls": true,
6062                 "stacking": {
6063                   "group": "A",
6064                   "mode": "none"
6065                 },
6066                 "thresholdsStyle": {
6067                   "mode": "off"
6068                 }
6069               },
6070               "links": [],
6071               "mappings": [],
6072               "thresholds": {
6073                 "mode": "absolute",
6074                 "steps": [
6075                   {
6076                     "color": "green"
6077                   },
6078                   {
6079                     "color": "red",
6080                     "value": 80
6081                   }
6082                 ]
6083               },
6084               "unit": "short"
6085             },
6086             "overrides": []
6087           },
6088           "gridPos": {
6089             "h": 8,
6090             "w": 12,
6091             "x": 12,
6092             "y": 33
6093           },
6094           "id": 118,
6095           "options": {
6096             "legend": {
6097               "calcs": [
6098                 "mean",
6099                 "lastNotNull",
6100                 "max"
6101               ],
6102               "displayMode": "table",
6103               "placement": "bottom",
6104               "showLegend": true
6105             },
6106             "tooltip": {
6107               "mode": "single"
6108             }
6109           },
6110           "pluginVersion": "8.1.3",
6111           "targets": [
6112             {
6113               "datasource": {
6114                 "type": "prometheus",
6115                 "uid": "PBFA97CFB590B2093"
6116               },
6117               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6118               "instant": false,
6119               "interval": "",
6120               "legendFormat": "stable",
6121               "refId": "A"
6122             },
6123             {
6124               "datasource": {
6125                 "type": "prometheus",
6126                 "uid": "PBFA97CFB590B2093"
6127               },
6128               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6129               "interval": "",
6130               "legendFormat": "preparing-rebalance",
6131               "refId": "B"
6132             },
6133             {
6134               "datasource": {
6135                 "type": "prometheus",
6136                 "uid": "PBFA97CFB590B2093"
6137               },
6138               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6139               "interval": "",
6140               "legendFormat": "dead",
6141               "refId": "C"
6142             },
6143             {
6144               "datasource": {
6145                 "type": "prometheus",
6146                 "uid": "PBFA97CFB590B2093"
6147               },
6148               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6149               "interval": "",
6150               "legendFormat": "completing-rebalance",
6151               "refId": "D"
6152             },
6153             {
6154               "datasource": {
6155                 "type": "prometheus",
6156                 "uid": "PBFA97CFB590B2093"
6157               },
6158               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6159               "interval": "",
6160               "legendFormat": "empty",
6161               "refId": "E"
6162             }
6163           ],
6164           "title": "Nb consumer groups per state",
6165           "type": "timeseries"
6166         }
6167       ],
6168       "targets": [
6169         {
6170           "datasource": {
6171             "type": "datasource",
6172             "uid": "grafana"
6173           },
6174           "refId": "A"
6175         }
6176       ],
6177       "title": "Group Coordinator",
6178       "type": "row"
6179     },
6180     {
6181       "collapsed": true,
6182       "datasource": {
6183         "type": "datasource",
6184         "uid": "grafana"
6185       },
6186       "gridPos": {
6187         "h": 1,
6188         "w": 24,
6189         "x": 0,
6190         "y": 54
6191       },
6192       "id": 46,
6193       "panels": [
6194         {
6195           "datasource": {
6196             "type": "prometheus",
6197             "uid": "PBFA97CFB590B2093"
6198           },
6199           "description": "The number of messages produced converted to match the log.message.format.version.",
6200           "fieldConfig": {
6201             "defaults": {
6202               "color": {
6203                 "mode": "palette-classic"
6204               },
6205               "custom": {
6206                 "axisLabel": "",
6207                 "axisPlacement": "auto",
6208                 "barAlignment": 0,
6209                 "drawStyle": "line",
6210                 "fillOpacity": 10,
6211                 "gradientMode": "none",
6212                 "hideFrom": {
6213                   "legend": false,
6214                   "tooltip": false,
6215                   "viz": false
6216                 },
6217                 "lineInterpolation": "linear",
6218                 "lineWidth": 1,
6219                 "pointSize": 5,
6220                 "scaleDistribution": {
6221                   "type": "linear"
6222                 },
6223                 "showPoints": "never",
6224                 "spanNulls": true,
6225                 "stacking": {
6226                   "group": "A",
6227                   "mode": "none"
6228                 },
6229                 "thresholdsStyle": {
6230                   "mode": "off"
6231                 }
6232               },
6233               "links": [],
6234               "mappings": [],
6235               "min": 0,
6236               "thresholds": {
6237                 "mode": "absolute",
6238                 "steps": [
6239                   {
6240                     "color": "green"
6241                   },
6242                   {
6243                     "color": "red",
6244                     "value": 80
6245                   }
6246                 ]
6247               },
6248               "unit": "ops"
6249             },
6250             "overrides": []
6251           },
6252           "gridPos": {
6253             "h": 8,
6254             "w": 8,
6255             "x": 0,
6256             "y": 34
6257           },
6258           "id": 48,
6259           "options": {
6260             "legend": {
6261               "calcs": [
6262                 "mean",
6263                 "lastNotNull",
6264                 "max"
6265               ],
6266               "displayMode": "list",
6267               "placement": "bottom",
6268               "showLegend": true
6269             },
6270             "tooltip": {
6271               "mode": "single"
6272             }
6273           },
6274           "pluginVersion": "8.1.3",
6275           "targets": [
6276             {
6277               "datasource": {
6278                 "type": "prometheus",
6279                 "uid": "PBFA97CFB590B2093"
6280               },
6281               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6282               "hide": false,
6283               "interval": "",
6284               "legendFormat": "{{topic}}",
6285               "refId": "A"
6286             }
6287           ],
6288           "title": "Number of produced message conversion",
6289           "type": "timeseries"
6290         },
6291         {
6292           "datasource": {
6293             "type": "prometheus",
6294             "uid": "PBFA97CFB590B2093"
6295           },
6296           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
6297           "fieldConfig": {
6298             "defaults": {
6299               "color": {
6300                 "mode": "palette-classic"
6301               },
6302               "custom": {
6303                 "axisLabel": "",
6304                 "axisPlacement": "auto",
6305                 "barAlignment": 0,
6306                 "drawStyle": "line",
6307                 "fillOpacity": 10,
6308                 "gradientMode": "none",
6309                 "hideFrom": {
6310                   "legend": false,
6311                   "tooltip": false,
6312                   "viz": false
6313                 },
6314                 "lineInterpolation": "linear",
6315                 "lineWidth": 1,
6316                 "pointSize": 5,
6317                 "scaleDistribution": {
6318                   "type": "linear"
6319                 },
6320                 "showPoints": "never",
6321                 "spanNulls": true,
6322                 "stacking": {
6323                   "group": "A",
6324                   "mode": "none"
6325                 },
6326                 "thresholdsStyle": {
6327                   "mode": "off"
6328                 }
6329               },
6330               "links": [],
6331               "mappings": [],
6332               "min": 0,
6333               "thresholds": {
6334                 "mode": "absolute",
6335                 "steps": [
6336                   {
6337                     "color": "green"
6338                   },
6339                   {
6340                     "color": "red",
6341                     "value": 80
6342                   }
6343                 ]
6344               },
6345               "unit": "ops"
6346             },
6347             "overrides": []
6348           },
6349           "gridPos": {
6350             "h": 8,
6351             "w": 8,
6352             "x": 8,
6353             "y": 34
6354           },
6355           "id": 51,
6356           "options": {
6357             "legend": {
6358               "calcs": [
6359                 "mean",
6360                 "lastNotNull",
6361                 "max"
6362               ],
6363               "displayMode": "list",
6364               "placement": "bottom",
6365               "showLegend": true
6366             },
6367             "tooltip": {
6368               "mode": "single"
6369             }
6370           },
6371           "pluginVersion": "8.1.3",
6372           "targets": [
6373             {
6374               "datasource": {
6375                 "type": "prometheus",
6376                 "uid": "PBFA97CFB590B2093"
6377               },
6378               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6379               "hide": false,
6380               "interval": "",
6381               "legendFormat": "{{topic}}",
6382               "refId": "A"
6383             }
6384           ],
6385           "title": "Number of consumed message conversion",
6386           "type": "timeseries"
6387         },
6388         {
6389           "datasource": {
6390             "type": "datasource",
6391             "uid": "grafana"
6392           },
6393           "description": "Number of connection per client version",
6394           "fieldConfig": {
6395             "defaults": {
6396               "color": {
6397                 "mode": "palette-classic"
6398               },
6399               "custom": {
6400                 "hideFrom": {
6401                   "legend": false,
6402                   "tooltip": false,
6403                   "viz": false
6404                 }
6405               },
6406               "decimals": 0,
6407               "mappings": [],
6408               "unit": "short"
6409             },
6410             "overrides": []
6411           },
6412           "gridPos": {
6413             "h": 8,
6414             "w": 8,
6415             "x": 16,
6416             "y": 34
6417           },
6418           "id": 96,
6419           "links": [],
6420           "options": {
6421             "legend": {
6422               "calcs": [],
6423               "displayMode": "table",
6424               "placement": "bottom",
6425               "showLegend": true,
6426               "values": [
6427                 "value"
6428               ]
6429             },
6430             "pieType": "pie",
6431             "reduceOptions": {
6432               "calcs": [
6433                 "lastNotNull"
6434               ],
6435               "fields": "",
6436               "values": false
6437             },
6438             "tooltip": {
6439               "mode": "single"
6440             }
6441           },
6442           "targets": [
6443             {
6444               "datasource": {
6445                 "type": "datasource",
6446                 "uid": "grafana"
6447               },
6448               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6449               "interval": "",
6450               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6451               "refId": "A"
6452             }
6453           ],
6454           "title": "Client version repartition",
6455           "type": "piechart"
6456         }
6457       ],
6458       "targets": [
6459         {
6460           "datasource": {
6461             "type": "datasource",
6462             "uid": "grafana"
6463           },
6464           "refId": "A"
6465         }
6466       ],
6467       "title": "Message Conversion",
6468       "type": "row"
6469     }
6470   ],
6471   "refresh": "5s",
6472   "schemaVersion": 38,
6473   "style": "dark",
6474   "tags": [],
6475   "templating": {
6476     "list": [
6477       {
6478         "current": {
6479           "selected": false,
6480           "text": "dev",
6481           "value": "dev"
6482         },
6483         "datasource": {
6484           "type": "prometheus",
6485           "uid": "PBFA97CFB590B2093"
6486         },
6487         "definition": "label_values(env)",
6488         "hide": 0,
6489         "includeAll": false,
6490         "label": "Environment",
6491         "multi": false,
6492         "name": "env",
6493         "options": [],
6494         "query": {
6495           "query": "label_values(env)",
6496           "refId": "Prometheus-env-Variable-Query"
6497         },
6498         "refresh": 1,
6499         "regex": "",
6500         "skipUrlSync": false,
6501         "sort": 0,
6502         "tagValuesQuery": "",
6503         "tagsQuery": "",
6504         "type": "query",
6505         "useTags": false
6506       },
6507       {
6508         "current": {
6509           "selected": true,
6510           "text": [
6511             "All"
6512           ],
6513           "value": [
6514             "$__all"
6515           ]
6516         },
6517         "datasource": {
6518           "type": "prometheus",
6519           "uid": "PBFA97CFB590B2093"
6520         },
6521         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6522         "hide": 0,
6523         "includeAll": true,
6524         "label": "Instance",
6525         "multi": true,
6526         "name": "instance",
6527         "options": [],
6528         "query": {
6529           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6530           "refId": "Prometheus-instance-Variable-Query"
6531         },
6532         "refresh": 1,
6533         "regex": "",
6534         "skipUrlSync": false,
6535         "sort": 0,
6536         "tagValuesQuery": "",
6537         "tagsQuery": "",
6538         "type": "query",
6539         "useTags": false
6540       },
6541       {
6542         "allValue": "",
6543         "current": {
6544           "selected": true,
6545           "text": [
6546             "0.95"
6547           ],
6548           "value": [
6549             "0.95"
6550           ]
6551         },
6552         "datasource": {
6553           "type": "prometheus",
6554           "uid": "PBFA97CFB590B2093"
6555         },
6556         "definition": "label_values(quantile)",
6557         "hide": 0,
6558         "includeAll": true,
6559         "label": "Percentile",
6560         "multi": true,
6561         "name": "percentile",
6562         "options": [],
6563         "query": {
6564           "query": "label_values(quantile)",
6565           "refId": "Prometheus-percentile-Variable-Query"
6566         },
6567         "refresh": 1,
6568         "regex": "",
6569         "skipUrlSync": false,
6570         "sort": 0,
6571         "tagValuesQuery": "",
6572         "tagsQuery": "",
6573         "type": "query",
6574         "useTags": false
6575       }
6576     ]
6577   },
6578   "time": {
6579     "from": "now-12h",
6580     "to": "now"
6581   },
6582   "timepicker": {
6583     "refresh_intervals": [
6584       "5s",
6585       "10s",
6586       "30s",
6587       "1m",
6588       "5m",
6589       "15m",
6590       "30m",
6591       "1h",
6592       "2h",
6593       "1d"
6594     ],
6595     "time_options": [
6596       "5m",
6597       "15m",
6598       "1h",
6599       "6h",
6600       "12h",
6601       "24h",
6602       "2d",
6603       "7d",
6604       "30d"
6605     ]
6606   },
6607   "timezone": "browser",
6608   "title": "Kafka cluster",
6609   "uid": "qu-QZdfZz",
6610   "version": 1,
6611   "weekStart": ""
6612 }