juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of Brokers Online",
  64       "fieldConfig": {
  65         "defaults": {
  66           "color": {
  67             "mode": "thresholds"
  68           },
  69           "mappings": [
  70             {
  71               "options": {
  72                 "match": "null",
  73                 "result": {
  74                   "text": "N/A"
  75                 }
  76               },
  77               "type": "special"
  78             }
  79           ],
  80           "thresholds": {
  81             "mode": "absolute",
  82             "steps": [
  83               {
  84                 "color": "#d44a3a",
  85                 "value": null
  86               },
  87               {
  88                 "color": "rgba(237, 129, 40, 0.89)",
  89                 "value": 0
  90               },
  91               {
  92                 "color": "#299c46",
  93                 "value": 2
  94               }
  95             ]
  96           },
  97           "unit": "none"
  98         },
  99         "overrides": []
 100       },
 101       "gridPos": {
 102         "h": 4,
 103         "w": 4,
 104         "x": 0,
 105         "y": 1
 106       },
 107       "id": 14,
 108       "links": [],
 109       "maxDataPoints": 100,
 110       "options": {
 111         "colorMode": "value",
 112         "graphMode": "none",
 113         "justifyMode": "auto",
 114         "orientation": "horizontal",
 115         "reduceOptions": {
 116           "calcs": [
 117             "lastNotNull"
 118           ],
 119           "fields": "",
 120           "values": false
 121         },
 122         "text": {},
 123         "textMode": "auto"
 124       },
 125       "pluginVersion": "10.0.3",
 126       "repeatDirection": "h",
 127       "targets": [
 128         {
 129           "datasource": {
 130             "type": "prometheus",
 131             "uid": "PBFA97CFB590B2093"
 132           },
 133           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 134           "format": "time_series",
 135           "instant": true,
 136           "interval": "",
 137           "intervalFactor": 1,
 138           "legendFormat": "",
 139           "refId": "A"
 140         }
 141       ],
 142       "title": "Brokers Online",
 143       "type": "stat"
 144     },
 145     {
 146       "datasource": {
 147         "type": "prometheus",
 148         "uid": "PBFA97CFB590B2093"
 149       },
 150       "description": "Number of active controllers in the cluster.",
 151       "fieldConfig": {
 152         "defaults": {
 153           "mappings": [
 154             {
 155               "options": {
 156                 "match": "null",
 157                 "result": {
 158                   "text": "N/A"
 159                 }
 160               },
 161               "type": "special"
 162             }
 163           ],
 164           "thresholds": {
 165             "mode": "absolute",
 166             "steps": [
 167               {
 168                 "color": "#299c46",
 169                 "value": null
 170               },
 171               {
 172                 "color": "#e5ac0e",
 173                 "value": 2
 174               },
 175               {
 176                 "color": "#bf1b00"
 177               }
 178             ]
 179           },
 180           "unit": "none"
 181         },
 182         "overrides": []
 183       },
 184       "gridPos": {
 185         "h": 4,
 186         "w": 5,
 187         "x": 4,
 188         "y": 1
 189       },
 190       "id": 12,
 191       "links": [],
 192       "maxDataPoints": 100,
 193       "options": {
 194         "colorMode": "value",
 195         "graphMode": "none",
 196         "justifyMode": "auto",
 197         "orientation": "vertical",
 198         "reduceOptions": {
 199           "calcs": [
 200             "lastNotNull"
 201           ],
 202           "fields": "",
 203           "values": false
 204         },
 205         "text": {},
 206         "textMode": "value_and_name"
 207       },
 208       "pluginVersion": "10.0.3",
 209       "targets": [
 210         {
 211           "datasource": {
 212             "type": "prometheus",
 213             "uid": "PBFA97CFB590B2093"
 214           },
 215           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 216           "format": "time_series",
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 1,
 220           "legendFormat": "{{instance}}",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Active Controllers",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Partitions that are online",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#d44a3a",
 254                 "value": null
 255               },
 256               {
 257                 "color": "rgba(237, 129, 40, 0.89)",
 258                 "value": 0
 259               },
 260               {
 261                 "color": "#299c46",
 262                 "value": 0
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 6,
 273         "x": 9,
 274         "y": 1
 275       },
 276       "id": 18,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Online Partitions",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 15,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Unclean leader election rate",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#299c46",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 2
 430               },
 431               {
 432                 "color": "#d44a3a"
 433               }
 434             ]
 435           },
 436           "unit": "none"
 437         },
 438         "overrides": []
 439       },
 440       "gridPos": {
 441         "h": 4,
 442         "w": 4,
 443         "x": 19,
 444         "y": 1
 445       },
 446       "id": 16,
 447       "links": [],
 448       "maxDataPoints": 100,
 449       "options": {
 450         "colorMode": "value",
 451         "graphMode": "none",
 452         "justifyMode": "auto",
 453         "orientation": "horizontal",
 454         "reduceOptions": {
 455           "calcs": [
 456             "lastNotNull"
 457           ],
 458           "fields": "",
 459           "values": false
 460         },
 461         "text": {},
 462         "textMode": "auto"
 463       },
 464       "pluginVersion": "10.0.3",
 465       "targets": [
 466         {
 467           "datasource": {
 468             "type": "prometheus",
 469             "uid": "PBFA97CFB590B2093"
 470           },
 471           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 472           "format": "time_series",
 473           "instant": true,
 474           "interval": "",
 475           "intervalFactor": 1,
 476           "legendFormat": "",
 477           "refId": "A"
 478         }
 479       ],
 480       "title": "Unclean Leader Election Rate",
 481       "type": "stat"
 482     },
 483     {
 484       "datasource": {
 485         "type": "prometheus",
 486         "uid": "PBFA97CFB590B2093"
 487       },
 488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 489       "fieldConfig": {
 490         "defaults": {
 491           "color": {
 492             "mode": "thresholds"
 493           },
 494           "mappings": [
 495             {
 496               "options": {
 497                 "match": "null",
 498                 "result": {
 499                   "text": "N/A"
 500                 }
 501               },
 502               "type": "special"
 503             }
 504           ],
 505           "thresholds": {
 506             "mode": "absolute",
 507             "steps": [
 508               {
 509                 "color": "#508642",
 510                 "value": null
 511               },
 512               {
 513                 "color": "rgba(237, 129, 40, 0.89)",
 514                 "value": 1
 515               },
 516               {
 517                 "color": "#bf1b00",
 518                 "value": 5
 519               }
 520             ]
 521           },
 522           "unit": "none"
 523         },
 524         "overrides": []
 525       },
 526       "gridPos": {
 527         "h": 4,
 528         "w": 4,
 529         "x": 0,
 530         "y": 5
 531       },
 532       "id": 20,
 533       "links": [],
 534       "maxDataPoints": 100,
 535       "options": {
 536         "colorMode": "value",
 537         "graphMode": "none",
 538         "justifyMode": "auto",
 539         "orientation": "horizontal",
 540         "reduceOptions": {
 541           "calcs": [
 542             "lastNotNull"
 543           ],
 544           "fields": "",
 545           "values": false
 546         },
 547         "text": {},
 548         "textMode": "auto"
 549       },
 550       "pluginVersion": "10.0.3",
 551       "targets": [
 552         {
 553           "datasource": {
 554             "type": "prometheus",
 555             "uid": "PBFA97CFB590B2093"
 556           },
 557           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 558           "format": "time_series",
 559           "hide": false,
 560           "instant": true,
 561           "interval": "",
 562           "intervalFactor": 2,
 563           "legendFormat": "",
 564           "refId": "A"
 565         }
 566       ],
 567       "title": "Under Replicated Partitions",
 568       "type": "stat"
 569     },
 570     {
 571       "datasource": {
 572         "type": "prometheus",
 573         "uid": "PBFA97CFB590B2093"
 574       },
 575       "description": "",
 576       "fieldConfig": {
 577         "defaults": {
 578           "color": {
 579             "mode": "palette-classic"
 580           },
 581           "custom": {
 582             "axisCenteredZero": false,
 583             "axisColorMode": "text",
 584             "axisLabel": "",
 585             "axisPlacement": "auto",
 586             "barAlignment": 0,
 587             "drawStyle": "line",
 588             "fillOpacity": 10,
 589             "gradientMode": "none",
 590             "hideFrom": {
 591               "legend": false,
 592               "tooltip": false,
 593               "viz": false
 594             },
 595             "lineInterpolation": "linear",
 596             "lineWidth": 1,
 597             "pointSize": 5,
 598             "scaleDistribution": {
 599               "type": "linear"
 600             },
 601             "showPoints": "never",
 602             "spanNulls": true,
 603             "stacking": {
 604               "group": "A",
 605               "mode": "none"
 606             },
 607             "thresholdsStyle": {
 608               "mode": "off"
 609             }
 610           },
 611           "decimals": 0,
 612           "links": [],
 613           "mappings": [],
 614           "min": 0,
 615           "thresholds": {
 616             "mode": "absolute",
 617             "steps": [
 618               {
 619                 "color": "green",
 620                 "value": null
 621               },
 622               {
 623                 "color": "red",
 624                 "value": 80
 625               }
 626             ]
 627           },
 628           "unit": "partitions"
 629         },
 630         "overrides": []
 631       },
 632       "gridPos": {
 633         "h": 12,
 634         "w": 11,
 635         "x": 4,
 636         "y": 5
 637       },
 638       "id": 123,
 639       "options": {
 640         "legend": {
 641           "calcs": [
 642             "lastNotNull",
 643             "max"
 644           ],
 645           "displayMode": "table",
 646           "placement": "bottom",
 647           "showLegend": true
 648         },
 649         "tooltip": {
 650           "mode": "single",
 651           "sort": "none"
 652         }
 653       },
 654       "pluginVersion": "8.1.3",
 655       "targets": [
 656         {
 657           "datasource": {
 658             "type": "prometheus",
 659             "uid": "PBFA97CFB590B2093"
 660           },
 661           "editorMode": "code",
 662           "exemplar": false,
 663           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 664           "hide": false,
 665           "interval": "",
 666           "legendFormat": "{{instance}}",
 667           "range": true,
 668           "refId": "A"
 669         }
 670       ],
 671       "title": "Underreplicated Partitions",
 672       "type": "timeseries"
 673     },
 674     {
 675       "datasource": {
 676         "type": "prometheus",
 677         "uid": "PBFA97CFB590B2093"
 678       },
 679       "description": "",
 680       "fieldConfig": {
 681         "defaults": {
 682           "color": {
 683             "mode": "palette-classic"
 684           },
 685           "custom": {
 686             "axisCenteredZero": false,
 687             "axisColorMode": "text",
 688             "axisLabel": "",
 689             "axisPlacement": "auto",
 690             "barAlignment": 0,
 691             "drawStyle": "line",
 692             "fillOpacity": 10,
 693             "gradientMode": "none",
 694             "hideFrom": {
 695               "legend": false,
 696               "tooltip": false,
 697               "viz": false
 698             },
 699             "lineInterpolation": "linear",
 700             "lineWidth": 1,
 701             "pointSize": 5,
 702             "scaleDistribution": {
 703               "type": "linear"
 704             },
 705             "showPoints": "never",
 706             "spanNulls": true,
 707             "stacking": {
 708               "group": "A",
 709               "mode": "none"
 710             },
 711             "thresholdsStyle": {
 712               "mode": "off"
 713             }
 714           },
 715           "decimals": 0,
 716           "links": [],
 717           "mappings": [
 718             {
 719               "options": {
 720                 "null": {
 721                   "index": 0,
 722                   "text": "0"
 723                 }
 724               },
 725               "type": "value"
 726             }
 727           ],
 728           "min": 0,
 729           "thresholds": {
 730             "mode": "absolute",
 731             "steps": [
 732               {
 733                 "color": "green",
 734                 "value": null
 735               },
 736               {
 737                 "color": "red",
 738                 "value": 80
 739               }
 740             ]
 741           },
 742           "unit": "partitions"
 743         },
 744         "overrides": []
 745       },
 746       "gridPos": {
 747         "h": 12,
 748         "w": 8,
 749         "x": 15,
 750         "y": 5
 751       },
 752       "id": 124,
 753       "options": {
 754         "legend": {
 755           "calcs": [
 756             "lastNotNull",
 757             "max"
 758           ],
 759           "displayMode": "table",
 760           "placement": "bottom",
 761           "showLegend": true
 762         },
 763         "tooltip": {
 764           "mode": "single",
 765           "sort": "none"
 766         }
 767       },
 768       "pluginVersion": "8.1.3",
 769       "targets": [
 770         {
 771           "datasource": {
 772             "type": "prometheus",
 773             "uid": "PBFA97CFB590B2093"
 774           },
 775           "editorMode": "code",
 776           "exemplar": false,
 777           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 778           "hide": false,
 779           "interval": "",
 780           "legendFormat": "{{instance}}",
 781           "range": true,
 782           "refId": "A"
 783         }
 784       ],
 785       "title": "Leader Count",
 786       "type": "timeseries"
 787     },
 788     {
 789       "datasource": {
 790         "type": "prometheus",
 791         "uid": "PBFA97CFB590B2093"
 792       },
 793       "description": "Number of partitions under min insync replicas.",
 794       "fieldConfig": {
 795         "defaults": {
 796           "color": {
 797             "mode": "thresholds"
 798           },
 799           "mappings": [
 800             {
 801               "options": {
 802                 "match": "null",
 803                 "result": {
 804                   "text": "N/A"
 805                 }
 806               },
 807               "type": "special"
 808             }
 809           ],
 810           "thresholds": {
 811             "mode": "absolute",
 812             "steps": [
 813               {
 814                 "color": "#508642",
 815                 "value": null
 816               },
 817               {
 818                 "color": "rgba(237, 129, 40, 0.89)",
 819                 "value": 1
 820               },
 821               {
 822                 "color": "#bf1b00",
 823                 "value": 5
 824               }
 825             ]
 826           },
 827           "unit": "none"
 828         },
 829         "overrides": []
 830       },
 831       "gridPos": {
 832         "h": 4,
 833         "w": 4,
 834         "x": 0,
 835         "y": 9
 836       },
 837       "id": 32,
 838       "links": [],
 839       "maxDataPoints": 100,
 840       "options": {
 841         "colorMode": "value",
 842         "graphMode": "none",
 843         "justifyMode": "auto",
 844         "orientation": "horizontal",
 845         "reduceOptions": {
 846           "calcs": [
 847             "lastNotNull"
 848           ],
 849           "fields": "",
 850           "values": false
 851         },
 852         "text": {},
 853         "textMode": "auto"
 854       },
 855       "pluginVersion": "10.0.3",
 856       "targets": [
 857         {
 858           "datasource": {
 859             "type": "prometheus",
 860             "uid": "PBFA97CFB590B2093"
 861           },
 862           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 863           "format": "time_series",
 864           "hide": false,
 865           "instant": true,
 866           "interval": "",
 867           "intervalFactor": 2,
 868           "legendFormat": "",
 869           "refId": "A"
 870         }
 871       ],
 872       "title": "Under Min ISR Partitions",
 873       "type": "stat"
 874     },
 875     {
 876       "datasource": {
 877         "type": "prometheus",
 878         "uid": "PBFA97CFB590B2093"
 879       },
 880       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 881       "fieldConfig": {
 882         "defaults": {
 883           "color": {
 884             "mode": "thresholds"
 885           },
 886           "mappings": [
 887             {
 888               "options": {
 889                 "match": "null",
 890                 "result": {
 891                   "text": "N/A"
 892                 }
 893               },
 894               "type": "special"
 895             }
 896           ],
 897           "thresholds": {
 898             "mode": "absolute",
 899             "steps": [
 900               {
 901                 "color": "#508642",
 902                 "value": null
 903               },
 904               {
 905                 "color": "#ef843c",
 906                 "value": 1
 907               },
 908               {
 909                 "color": "#bf1b00",
 910                 "value": 1
 911               }
 912             ]
 913           },
 914           "unit": "none"
 915         },
 916         "overrides": []
 917       },
 918       "gridPos": {
 919         "h": 4,
 920         "w": 4,
 921         "x": 0,
 922         "y": 13
 923       },
 924       "id": 22,
 925       "links": [],
 926       "maxDataPoints": 100,
 927       "options": {
 928         "colorMode": "value",
 929         "graphMode": "none",
 930         "justifyMode": "auto",
 931         "orientation": "horizontal",
 932         "reduceOptions": {
 933           "calcs": [
 934             "lastNotNull"
 935           ],
 936           "fields": "",
 937           "values": false
 938         },
 939         "text": {},
 940         "textMode": "auto"
 941       },
 942       "pluginVersion": "10.0.3",
 943       "targets": [
 944         {
 945           "datasource": {
 946             "type": "prometheus",
 947             "uid": "PBFA97CFB590B2093"
 948           },
 949           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 950           "format": "time_series",
 951           "instant": true,
 952           "interval": "",
 953           "intervalFactor": 1,
 954           "legendFormat": "",
 955           "refId": "A"
 956         }
 957       ],
 958       "title": "Offline Partitions Count",
 959       "type": "stat"
 960     },
 961     {
 962       "collapsed": true,
 963       "datasource": {
 964         "type": "datasource",
 965         "uid": "grafana"
 966       },
 967       "gridPos": {
 968         "h": 1,
 969         "w": 24,
 970         "x": 0,
 971         "y": 17
 972       },
 973       "id": 40,
 974       "panels": [
 975         {
 976           "datasource": {
 977             "type": "prometheus",
 978             "uid": "PBFA97CFB590B2093"
 979           },
 980           "fieldConfig": {
 981             "defaults": {
 982               "color": {
 983                 "mode": "palette-classic"
 984               },
 985               "custom": {
 986                 "axisCenteredZero": false,
 987                 "axisColorMode": "text",
 988                 "axisLabel": "Cores",
 989                 "axisPlacement": "auto",
 990                 "barAlignment": 0,
 991                 "drawStyle": "line",
 992                 "fillOpacity": 10,
 993                 "gradientMode": "none",
 994                 "hideFrom": {
 995                   "legend": false,
 996                   "tooltip": false,
 997                   "viz": false
 998                 },
 999                 "lineInterpolation": "linear",
1000                 "lineWidth": 2,
1001                 "pointSize": 5,
1002                 "scaleDistribution": {
1003                   "type": "linear"
1004                 },
1005                 "showPoints": "never",
1006                 "spanNulls": false,
1007                 "stacking": {
1008                   "group": "A",
1009                   "mode": "none"
1010                 },
1011                 "thresholdsStyle": {
1012                   "mode": "off"
1013                 }
1014               },
1015               "links": [],
1016               "mappings": [],
1017               "thresholds": {
1018                 "mode": "absolute",
1019                 "steps": [
1020                   {
1021                     "color": "green"
1022                   },
1023                   {
1024                     "color": "red",
1025                     "value": 80
1026                   }
1027                 ]
1028               },
1029               "unit": "percent"
1030             },
1031             "overrides": [
1032               {
1033                 "matcher": {
1034                   "id": "byName",
1035                   "options": "localhost:7071"
1036                 },
1037                 "properties": [
1038                   {
1039                     "id": "color",
1040                     "value": {
1041                       "fixedColor": "#629E51",
1042                       "mode": "fixed"
1043                     }
1044                   }
1045                 ]
1046               }
1047             ]
1048           },
1049           "gridPos": {
1050             "h": 12,
1051             "w": 8,
1052             "x": 0,
1053             "y": 19
1054           },
1055           "id": 27,
1056           "links": [],
1057           "options": {
1058             "legend": {
1059               "calcs": [
1060                 "mean",
1061                 "lastNotNull",
1062                 "max"
1063               ],
1064               "displayMode": "table",
1065               "placement": "bottom",
1066               "showLegend": true
1067             },
1068             "tooltip": {
1069               "mode": "single",
1070               "sort": "none"
1071             }
1072           },
1073           "pluginVersion": "8.1.3",
1074           "targets": [
1075             {
1076               "datasource": {
1077                 "type": "prometheus",
1078                 "uid": "PBFA97CFB590B2093"
1079               },
1080               "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1081               "format": "time_series",
1082               "interval": "",
1083               "intervalFactor": 2,
1084               "legendFormat": "{{instance}}",
1085               "metric": "process_cpu_secondspersec",
1086               "refId": "A",
1087               "step": 4
1088             }
1089           ],
1090           "title": "CPU Usage",
1091           "type": "timeseries"
1092         },
1093         {
1094           "datasource": {
1095             "type": "prometheus",
1096             "uid": "PBFA97CFB590B2093"
1097           },
1098           "fieldConfig": {
1099             "defaults": {
1100               "color": {
1101                 "mode": "palette-classic"
1102               },
1103               "custom": {
1104                 "axisCenteredZero": false,
1105                 "axisColorMode": "text",
1106                 "axisLabel": "Memory",
1107                 "axisPlacement": "auto",
1108                 "barAlignment": 0,
1109                 "drawStyle": "line",
1110                 "fillOpacity": 10,
1111                 "gradientMode": "none",
1112                 "hideFrom": {
1113                   "legend": false,
1114                   "tooltip": false,
1115                   "viz": false
1116                 },
1117                 "lineInterpolation": "linear",
1118                 "lineWidth": 2,
1119                 "pointSize": 5,
1120                 "scaleDistribution": {
1121                   "type": "linear"
1122                 },
1123                 "showPoints": "never",
1124                 "spanNulls": false,
1125                 "stacking": {
1126                   "group": "A",
1127                   "mode": "none"
1128                 },
1129                 "thresholdsStyle": {
1130                   "mode": "off"
1131                 }
1132               },
1133               "links": [],
1134               "mappings": [],
1135               "thresholds": {
1136                 "mode": "absolute",
1137                 "steps": [
1138                   {
1139                     "color": "green"
1140                   },
1141                   {
1142                     "color": "red",
1143                     "value": 80
1144                   }
1145                 ]
1146               },
1147               "unit": "bytes"
1148             },
1149             "overrides": [
1150               {
1151                 "matcher": {
1152                   "id": "byName",
1153                   "options": "localhost:7071"
1154                 },
1155                 "properties": [
1156                   {
1157                     "id": "color",
1158                     "value": {
1159                       "fixedColor": "#BA43A9",
1160                       "mode": "fixed"
1161                     }
1162                   }
1163                 ]
1164               },
1165               {
1166                 "__systemRef": "hideSeriesFrom",
1167                 "matcher": {
1168                   "id": "byNames",
1169                   "options": {
1170                     "mode": "exclude",
1171                     "names": [
1172                       "kafka-2:80"
1173                     ],
1174                     "prefix": "All except:",
1175                     "readOnly": true
1176                   }
1177                 },
1178                 "properties": [
1179                   {
1180                     "id": "custom.hideFrom",
1181                     "value": {
1182                       "legend": false,
1183                       "tooltip": false,
1184                       "viz": true
1185                     }
1186                   }
1187                 ]
1188               }
1189             ]
1190           },
1191           "gridPos": {
1192             "h": 12,
1193             "w": 8,
1194             "x": 8,
1195             "y": 19
1196           },
1197           "id": 2,
1198           "links": [],
1199           "options": {
1200             "legend": {
1201               "calcs": [
1202                 "mean",
1203                 "lastNotNull",
1204                 "max"
1205               ],
1206               "displayMode": "table",
1207               "placement": "bottom",
1208               "showLegend": true
1209             },
1210             "tooltip": {
1211               "mode": "single",
1212               "sort": "none"
1213             }
1214           },
1215           "pluginVersion": "8.1.3",
1216           "targets": [
1217             {
1218               "datasource": {
1219                 "type": "prometheus",
1220                 "uid": "PBFA97CFB590B2093"
1221               },
1222               "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1223               "interval": "",
1224               "intervalFactor": 2,
1225               "legendFormat": "{{instance}}",
1226               "metric": "jvm_memory_bytes_used",
1227               "refId": "A",
1228               "step": 4
1229             },
1230             {
1231               "datasource": {
1232                 "type": "prometheus",
1233                 "uid": "PBFA97CFB590B2093"
1234               },
1235               "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1236               "interval": "",
1237               "legendFormat": "{{instance}}",
1238               "refId": "B"
1239             }
1240           ],
1241           "title": "JVM Memory Used",
1242           "type": "timeseries"
1243         },
1244         {
1245           "datasource": {
1246             "type": "prometheus",
1247             "uid": "PBFA97CFB590B2093"
1248           },
1249           "fieldConfig": {
1250             "defaults": {
1251               "color": {
1252                 "mode": "palette-classic"
1253               },
1254               "custom": {
1255                 "axisCenteredZero": false,
1256                 "axisColorMode": "text",
1257                 "axisLabel": "% time in GC",
1258                 "axisPlacement": "auto",
1259                 "barAlignment": 0,
1260                 "drawStyle": "line",
1261                 "fillOpacity": 10,
1262                 "gradientMode": "none",
1263                 "hideFrom": {
1264                   "legend": false,
1265                   "tooltip": false,
1266                   "viz": false
1267                 },
1268                 "lineInterpolation": "linear",
1269                 "lineWidth": 2,
1270                 "pointSize": 5,
1271                 "scaleDistribution": {
1272                   "type": "linear"
1273                 },
1274                 "showPoints": "never",
1275                 "spanNulls": false,
1276                 "stacking": {
1277                   "group": "A",
1278                   "mode": "none"
1279                 },
1280                 "thresholdsStyle": {
1281                   "mode": "off"
1282                 }
1283               },
1284               "links": [],
1285               "mappings": [],
1286               "thresholds": {
1287                 "mode": "absolute",
1288                 "steps": [
1289                   {
1290                     "color": "green"
1291                   },
1292                   {
1293                     "color": "red",
1294                     "value": 80
1295                   }
1296                 ]
1297               },
1298               "unit": "percentunit"
1299             },
1300             "overrides": [
1301               {
1302                 "matcher": {
1303                   "id": "byName",
1304                   "options": "localhost:7071"
1305                 },
1306                 "properties": [
1307                   {
1308                     "id": "color",
1309                     "value": {
1310                       "fixedColor": "#890F02",
1311                       "mode": "fixed"
1312                     }
1313                   }
1314                 ]
1315               }
1316             ]
1317           },
1318           "gridPos": {
1319             "h": 12,
1320             "w": 8,
1321             "x": 16,
1322             "y": 19
1323           },
1324           "id": 3,
1325           "links": [],
1326           "options": {
1327             "legend": {
1328               "calcs": [
1329                 "mean",
1330                 "lastNotNull",
1331                 "max"
1332               ],
1333               "displayMode": "table",
1334               "placement": "bottom",
1335               "showLegend": true
1336             },
1337             "tooltip": {
1338               "mode": "single",
1339               "sort": "none"
1340             }
1341           },
1342           "pluginVersion": "8.1.3",
1343           "targets": [
1344             {
1345               "datasource": {
1346                 "type": "prometheus",
1347                 "uid": "PBFA97CFB590B2093"
1348               },
1349               "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1350               "interval": "",
1351               "intervalFactor": 2,
1352               "legendFormat": "{{instance}}",
1353               "metric": "jvm_gc_collection_seconds_sum",
1354               "refId": "A",
1355               "step": 4
1356             }
1357           ],
1358           "title": "Time spent in GC",
1359           "type": "timeseries"
1360         }
1361       ],
1362       "targets": [
1363         {
1364           "datasource": {
1365             "type": "datasource",
1366             "uid": "grafana"
1367           },
1368           "refId": "A"
1369         }
1370       ],
1371       "title": "System",
1372       "type": "row"
1373     },
1374     {
1375       "collapsed": false,
1376       "datasource": {
1377         "type": "datasource",
1378         "uid": "grafana"
1379       },
1380       "gridPos": {
1381         "h": 1,
1382         "w": 24,
1383         "x": 0,
1384         "y": 18
1385       },
1386       "id": 31,
1387       "panels": [],
1388       "targets": [
1389         {
1390           "datasource": {
1391             "type": "datasource",
1392             "uid": "grafana"
1393           },
1394           "refId": "A"
1395         }
1396       ],
1397       "title": "Throughput & Request rate",
1398       "type": "row"
1399     },
1400     {
1401       "datasource": {
1402         "type": "prometheus",
1403         "uid": "PBFA97CFB590B2093"
1404       },
1405       "fieldConfig": {
1406         "defaults": {
1407           "color": {
1408             "mode": "palette-classic"
1409           },
1410           "custom": {
1411             "axisCenteredZero": false,
1412             "axisColorMode": "text",
1413             "axisLabel": "Bytes/s",
1414             "axisPlacement": "auto",
1415             "barAlignment": 0,
1416             "drawStyle": "line",
1417             "fillOpacity": 10,
1418             "gradientMode": "none",
1419             "hideFrom": {
1420               "legend": false,
1421               "tooltip": false,
1422               "viz": false
1423             },
1424             "lineInterpolation": "linear",
1425             "lineWidth": 2,
1426             "pointSize": 5,
1427             "scaleDistribution": {
1428               "type": "linear"
1429             },
1430             "showPoints": "never",
1431             "spanNulls": false,
1432             "stacking": {
1433               "group": "A",
1434               "mode": "none"
1435             },
1436             "thresholdsStyle": {
1437               "mode": "off"
1438             }
1439           },
1440           "links": [],
1441           "mappings": [],
1442           "thresholds": {
1443             "mode": "absolute",
1444             "steps": [
1445               {
1446                 "color": "green",
1447                 "value": null
1448               },
1449               {
1450                 "color": "red",
1451                 "value": 80
1452               }
1453             ]
1454           },
1455           "unit": "Bps"
1456         },
1457         "overrides": []
1458       },
1459       "gridPos": {
1460         "h": 8,
1461         "w": 12,
1462         "x": 0,
1463         "y": 19
1464       },
1465       "id": 84,
1466       "links": [],
1467       "options": {
1468         "legend": {
1469           "calcs": [
1470             "mean",
1471             "lastNotNull",
1472             "max"
1473           ],
1474           "displayMode": "list",
1475           "placement": "bottom",
1476           "showLegend": true
1477         },
1478         "tooltip": {
1479           "mode": "single",
1480           "sort": "none"
1481         }
1482       },
1483       "pluginVersion": "8.1.3",
1484       "targets": [
1485         {
1486           "datasource": {
1487             "type": "prometheus",
1488             "uid": "PBFA97CFB590B2093"
1489           },
1490           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1491           "format": "time_series",
1492           "interval": "",
1493           "intervalFactor": 2,
1494           "legendFormat": "Bytes in",
1495           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1496           "refId": "A",
1497           "step": 4
1498         },
1499         {
1500           "datasource": {
1501             "type": "prometheus",
1502             "uid": "PBFA97CFB590B2093"
1503           },
1504           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1505           "format": "time_series",
1506           "hide": false,
1507           "interval": "",
1508           "intervalFactor": 2,
1509           "legendFormat": "Bytes out",
1510           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1511           "refId": "B",
1512           "step": 4
1513         }
1514       ],
1515       "title": "Broker network throughput",
1516       "type": "timeseries"
1517     },
1518     {
1519       "datasource": {
1520         "type": "prometheus",
1521         "uid": "PBFA97CFB590B2093"
1522       },
1523       "description": "Produce request rate.",
1524       "fieldConfig": {
1525         "defaults": {
1526           "mappings": [
1527             {
1528               "options": {
1529                 "match": "null",
1530                 "result": {
1531                   "text": "N/A"
1532                 }
1533               },
1534               "type": "special"
1535             }
1536           ],
1537           "thresholds": {
1538             "mode": "absolute",
1539             "steps": [
1540               {
1541                 "color": "green",
1542                 "value": null
1543               }
1544             ]
1545           },
1546           "unit": "none"
1547         },
1548         "overrides": []
1549       },
1550       "gridPos": {
1551         "h": 4,
1552         "w": 4,
1553         "x": 12,
1554         "y": 19
1555       },
1556       "id": 93,
1557       "links": [],
1558       "maxDataPoints": 100,
1559       "options": {
1560         "colorMode": "value",
1561         "graphMode": "area",
1562         "justifyMode": "auto",
1563         "orientation": "horizontal",
1564         "reduceOptions": {
1565           "calcs": [
1566             "lastNotNull"
1567           ],
1568           "fields": "",
1569           "values": false
1570         },
1571         "text": {},
1572         "textMode": "auto"
1573       },
1574       "pluginVersion": "10.0.3",
1575       "targets": [
1576         {
1577           "datasource": {
1578             "type": "prometheus",
1579             "uid": "PBFA97CFB590B2093"
1580           },
1581           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1582           "interval": "",
1583           "legendFormat": "",
1584           "refId": "A"
1585         }
1586       ],
1587       "title": "All Request Per Sec",
1588       "type": "stat"
1589     },
1590     {
1591       "datasource": {
1592         "type": "prometheus",
1593         "uid": "PBFA97CFB590B2093"
1594       },
1595       "description": "Fetch request rate.",
1596       "fieldConfig": {
1597         "defaults": {
1598           "mappings": [
1599             {
1600               "options": {
1601                 "match": "null",
1602                 "result": {
1603                   "text": "N/A"
1604                 }
1605               },
1606               "type": "special"
1607             }
1608           ],
1609           "thresholds": {
1610             "mode": "absolute",
1611             "steps": [
1612               {
1613                 "color": "green",
1614                 "value": null
1615               }
1616             ]
1617           },
1618           "unit": "none"
1619         },
1620         "overrides": []
1621       },
1622       "gridPos": {
1623         "h": 4,
1624         "w": 4,
1625         "x": 16,
1626         "y": 19
1627       },
1628       "id": 37,
1629       "links": [],
1630       "maxDataPoints": 100,
1631       "options": {
1632         "colorMode": "value",
1633         "graphMode": "area",
1634         "justifyMode": "auto",
1635         "orientation": "horizontal",
1636         "reduceOptions": {
1637           "calcs": [
1638             "lastNotNull"
1639           ],
1640           "fields": "",
1641           "values": false
1642         },
1643         "text": {},
1644         "textMode": "auto"
1645       },
1646       "pluginVersion": "10.0.3",
1647       "targets": [
1648         {
1649           "datasource": {
1650             "type": "prometheus",
1651             "uid": "PBFA97CFB590B2093"
1652           },
1653           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1654           "interval": "",
1655           "legendFormat": "",
1656           "refId": "A"
1657         }
1658       ],
1659       "title": "Consumer Fetch Request Per Sec",
1660       "type": "stat"
1661     },
1662     {
1663       "datasource": {
1664         "type": "prometheus",
1665         "uid": "PBFA97CFB590B2093"
1666       },
1667       "description": "Produce request rate.",
1668       "fieldConfig": {
1669         "defaults": {
1670           "mappings": [
1671             {
1672               "options": {
1673                 "match": "null",
1674                 "result": {
1675                   "text": "N/A"
1676                 }
1677               },
1678               "type": "special"
1679             }
1680           ],
1681           "thresholds": {
1682             "mode": "absolute",
1683             "steps": [
1684               {
1685                 "color": "green",
1686                 "value": null
1687               }
1688             ]
1689           },
1690           "unit": "none"
1691         },
1692         "overrides": []
1693       },
1694       "gridPos": {
1695         "h": 4,
1696         "w": 4,
1697         "x": 20,
1698         "y": 19
1699       },
1700       "id": 35,
1701       "links": [],
1702       "maxDataPoints": 100,
1703       "options": {
1704         "colorMode": "value",
1705         "graphMode": "area",
1706         "justifyMode": "auto",
1707         "orientation": "horizontal",
1708         "reduceOptions": {
1709           "calcs": [
1710             "lastNotNull"
1711           ],
1712           "fields": "",
1713           "values": false
1714         },
1715         "text": {},
1716         "textMode": "auto"
1717       },
1718       "pluginVersion": "10.0.3",
1719       "targets": [
1720         {
1721           "datasource": {
1722             "type": "prometheus",
1723             "uid": "PBFA97CFB590B2093"
1724           },
1725           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1726           "interval": "",
1727           "legendFormat": "",
1728           "refId": "A"
1729         }
1730       ],
1731       "title": "Produce Request Per Sec",
1732       "type": "stat"
1733     },
1734     {
1735       "datasource": {
1736         "type": "prometheus",
1737         "uid": "PBFA97CFB590B2093"
1738       },
1739       "description": "Metadata request rate.",
1740       "fieldConfig": {
1741         "defaults": {
1742           "mappings": [
1743             {
1744               "options": {
1745                 "match": "null",
1746                 "result": {
1747                   "text": "N/A"
1748                 }
1749               },
1750               "type": "special"
1751             }
1752           ],
1753           "thresholds": {
1754             "mode": "absolute",
1755             "steps": [
1756               {
1757                 "color": "green",
1758                 "value": null
1759               }
1760             ]
1761           },
1762           "unit": "none"
1763         },
1764         "overrides": []
1765       },
1766       "gridPos": {
1767         "h": 4,
1768         "w": 4,
1769         "x": 12,
1770         "y": 23
1771       },
1772       "id": 36,
1773       "links": [],
1774       "maxDataPoints": 100,
1775       "options": {
1776         "colorMode": "value",
1777         "graphMode": "area",
1778         "justifyMode": "auto",
1779         "orientation": "horizontal",
1780         "reduceOptions": {
1781           "calcs": [
1782             "lastNotNull"
1783           ],
1784           "fields": "",
1785           "values": false
1786         },
1787         "text": {},
1788         "textMode": "auto"
1789       },
1790       "pluginVersion": "10.0.3",
1791       "targets": [
1792         {
1793           "datasource": {
1794             "type": "prometheus",
1795             "uid": "PBFA97CFB590B2093"
1796           },
1797           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1798           "interval": "",
1799           "legendFormat": "",
1800           "refId": "A"
1801         }
1802       ],
1803       "title": "Metadata Request Per Sec",
1804       "type": "stat"
1805     },
1806     {
1807       "datasource": {
1808         "type": "prometheus",
1809         "uid": "PBFA97CFB590B2093"
1810       },
1811       "description": "Offset Commit request rate.",
1812       "fieldConfig": {
1813         "defaults": {
1814           "mappings": [
1815             {
1816               "options": {
1817                 "match": "null",
1818                 "result": {
1819                   "text": "N/A"
1820                 }
1821               },
1822               "type": "special"
1823             }
1824           ],
1825           "thresholds": {
1826             "mode": "absolute",
1827             "steps": [
1828               {
1829                 "color": "green",
1830                 "value": null
1831               }
1832             ]
1833           },
1834           "unit": "none"
1835         },
1836         "overrides": []
1837       },
1838       "gridPos": {
1839         "h": 4,
1840         "w": 4,
1841         "x": 16,
1842         "y": 23
1843       },
1844       "id": 38,
1845       "links": [],
1846       "maxDataPoints": 100,
1847       "options": {
1848         "colorMode": "value",
1849         "graphMode": "area",
1850         "justifyMode": "auto",
1851         "orientation": "horizontal",
1852         "reduceOptions": {
1853           "calcs": [
1854             "lastNotNull"
1855           ],
1856           "fields": "",
1857           "values": false
1858         },
1859         "text": {},
1860         "textMode": "auto"
1861       },
1862       "pluginVersion": "10.0.3",
1863       "targets": [
1864         {
1865           "datasource": {
1866             "type": "prometheus",
1867             "uid": "PBFA97CFB590B2093"
1868           },
1869           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1870           "interval": "",
1871           "legendFormat": "",
1872           "refId": "A"
1873         }
1874       ],
1875       "title": "Offset Commit Request Per Sec",
1876       "type": "stat"
1877     },
1878     {
1879       "datasource": {
1880         "type": "prometheus",
1881         "uid": "PBFA97CFB590B2093"
1882       },
1883       "description": "Fetch request rate.",
1884       "fieldConfig": {
1885         "defaults": {
1886           "mappings": [
1887             {
1888               "options": {
1889                 "match": "null",
1890                 "result": {
1891                   "text": "N/A"
1892                 }
1893               },
1894               "type": "special"
1895             }
1896           ],
1897           "thresholds": {
1898             "mode": "absolute",
1899             "steps": [
1900               {
1901                 "color": "green",
1902                 "value": null
1903               }
1904             ]
1905           },
1906           "unit": "none"
1907         },
1908         "overrides": []
1909       },
1910       "gridPos": {
1911         "h": 4,
1912         "w": 4,
1913         "x": 20,
1914         "y": 23
1915       },
1916       "id": 94,
1917       "links": [],
1918       "maxDataPoints": 100,
1919       "options": {
1920         "colorMode": "value",
1921         "graphMode": "area",
1922         "justifyMode": "auto",
1923         "orientation": "horizontal",
1924         "reduceOptions": {
1925           "calcs": [
1926             "lastNotNull"
1927           ],
1928           "fields": "",
1929           "values": false
1930         },
1931         "text": {},
1932         "textMode": "auto"
1933       },
1934       "pluginVersion": "10.0.3",
1935       "targets": [
1936         {
1937           "datasource": {
1938             "type": "prometheus",
1939             "uid": "PBFA97CFB590B2093"
1940           },
1941           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1942           "interval": "",
1943           "legendFormat": "",
1944           "refId": "A"
1945         }
1946       ],
1947       "title": "Broker Fetch Request Per Sec",
1948       "type": "stat"
1949     },
1950     {
1951       "collapsed": false,
1952       "datasource": {
1953         "type": "datasource",
1954         "uid": "grafana"
1955       },
1956       "gridPos": {
1957         "h": 1,
1958         "w": 24,
1959         "x": 0,
1960         "y": 27
1961       },
1962       "id": 29,
1963       "panels": [],
1964       "targets": [
1965         {
1966           "datasource": {
1967             "type": "datasource",
1968             "uid": "grafana"
1969           },
1970           "refId": "A"
1971         }
1972       ],
1973       "title": "Throughput In/Out",
1974       "type": "row"
1975     },
1976     {
1977       "datasource": {
1978         "type": "prometheus",
1979         "uid": "PBFA97CFB590B2093"
1980       },
1981       "fieldConfig": {
1982         "defaults": {
1983           "color": {
1984             "mode": "palette-classic"
1985           },
1986           "custom": {
1987             "axisCenteredZero": false,
1988             "axisColorMode": "text",
1989             "axisLabel": "Messages/s",
1990             "axisPlacement": "auto",
1991             "barAlignment": 0,
1992             "drawStyle": "line",
1993             "fillOpacity": 10,
1994             "gradientMode": "none",
1995             "hideFrom": {
1996               "legend": false,
1997               "tooltip": false,
1998               "viz": false
1999             },
2000             "lineInterpolation": "linear",
2001             "lineWidth": 2,
2002             "pointSize": 5,
2003             "scaleDistribution": {
2004               "type": "linear"
2005             },
2006             "showPoints": "never",
2007             "spanNulls": false,
2008             "stacking": {
2009               "group": "A",
2010               "mode": "none"
2011             },
2012             "thresholdsStyle": {
2013               "mode": "off"
2014             }
2015           },
2016           "links": [],
2017           "mappings": [],
2018           "min": 0,
2019           "thresholds": {
2020             "mode": "absolute",
2021             "steps": [
2022               {
2023                 "color": "green",
2024                 "value": null
2025               }
2026             ]
2027           },
2028           "unit": "cps"
2029         },
2030         "overrides": []
2031       },
2032       "gridPos": {
2033         "h": 8,
2034         "w": 8,
2035         "x": 0,
2036         "y": 28
2037       },
2038       "id": 4,
2039       "links": [],
2040       "options": {
2041         "legend": {
2042           "calcs": [
2043             "mean",
2044             "lastNotNull",
2045             "max"
2046           ],
2047           "displayMode": "table",
2048           "placement": "bottom",
2049           "showLegend": true
2050         },
2051         "tooltip": {
2052           "mode": "single",
2053           "sort": "none"
2054         }
2055       },
2056       "pluginVersion": "8.1.3",
2057       "targets": [
2058         {
2059           "datasource": {
2060             "type": "prometheus",
2061             "uid": "PBFA97CFB590B2093"
2062           },
2063           "exemplar": true,
2064           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2065           "interval": "",
2066           "intervalFactor": 2,
2067           "legendFormat": "bytes/sec",
2068           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2069           "refId": "A",
2070           "step": 4
2071         }
2072       ],
2073       "title": "Messages In",
2074       "type": "timeseries"
2075     },
2076     {
2077       "datasource": {
2078         "type": "prometheus",
2079         "uid": "PBFA97CFB590B2093"
2080       },
2081       "fieldConfig": {
2082         "defaults": {
2083           "color": {
2084             "mode": "palette-classic"
2085           },
2086           "custom": {
2087             "axisCenteredZero": false,
2088             "axisColorMode": "text",
2089             "axisLabel": "Bytes/s",
2090             "axisPlacement": "auto",
2091             "barAlignment": 0,
2092             "drawStyle": "line",
2093             "fillOpacity": 10,
2094             "gradientMode": "none",
2095             "hideFrom": {
2096               "legend": false,
2097               "tooltip": false,
2098               "viz": false
2099             },
2100             "lineInterpolation": "linear",
2101             "lineWidth": 2,
2102             "pointSize": 5,
2103             "scaleDistribution": {
2104               "type": "linear"
2105             },
2106             "showPoints": "never",
2107             "spanNulls": false,
2108             "stacking": {
2109               "group": "A",
2110               "mode": "none"
2111             },
2112             "thresholdsStyle": {
2113               "mode": "off"
2114             }
2115           },
2116           "links": [],
2117           "mappings": [],
2118           "min": 0,
2119           "thresholds": {
2120             "mode": "absolute",
2121             "steps": [
2122               {
2123                 "color": "green",
2124                 "value": null
2125               },
2126               {
2127                 "color": "red",
2128                 "value": 80
2129               }
2130             ]
2131           },
2132           "unit": "Bps"
2133         },
2134         "overrides": []
2135       },
2136       "gridPos": {
2137         "h": 8,
2138         "w": 8,
2139         "x": 8,
2140         "y": 28
2141       },
2142       "id": 5,
2143       "links": [],
2144       "options": {
2145         "legend": {
2146           "calcs": [
2147             "mean",
2148             "lastNotNull",
2149             "max"
2150           ],
2151           "displayMode": "table",
2152           "placement": "bottom",
2153           "showLegend": true
2154         },
2155         "tooltip": {
2156           "mode": "single",
2157           "sort": "none"
2158         }
2159       },
2160       "pluginVersion": "8.1.3",
2161       "targets": [
2162         {
2163           "datasource": {
2164             "type": "prometheus",
2165             "uid": "PBFA97CFB590B2093"
2166           },
2167           "exemplar": true,
2168           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2169           "format": "time_series",
2170           "interval": "",
2171           "intervalFactor": 2,
2172           "legendFormat": "bytes/sec",
2173           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2174           "refId": "A",
2175           "step": 4
2176         }
2177       ],
2178       "title": "Bytes In",
2179       "type": "timeseries"
2180     },
2181     {
2182       "datasource": {
2183         "type": "prometheus",
2184         "uid": "PBFA97CFB590B2093"
2185       },
2186       "fieldConfig": {
2187         "defaults": {
2188           "color": {
2189             "mode": "palette-classic"
2190           },
2191           "custom": {
2192             "axisCenteredZero": false,
2193             "axisColorMode": "text",
2194             "axisLabel": "Bytes/s",
2195             "axisPlacement": "auto",
2196             "barAlignment": 0,
2197             "drawStyle": "line",
2198             "fillOpacity": 10,
2199             "gradientMode": "none",
2200             "hideFrom": {
2201               "legend": false,
2202               "tooltip": false,
2203               "viz": false
2204             },
2205             "lineInterpolation": "linear",
2206             "lineWidth": 2,
2207             "pointSize": 5,
2208             "scaleDistribution": {
2209               "type": "linear"
2210             },
2211             "showPoints": "never",
2212             "spanNulls": false,
2213             "stacking": {
2214               "group": "A",
2215               "mode": "none"
2216             },
2217             "thresholdsStyle": {
2218               "mode": "off"
2219             }
2220           },
2221           "links": [],
2222           "mappings": [],
2223           "min": 0,
2224           "thresholds": {
2225             "mode": "absolute",
2226             "steps": [
2227               {
2228                 "color": "green",
2229                 "value": null
2230               },
2231               {
2232                 "color": "red",
2233                 "value": 80
2234               }
2235             ]
2236           },
2237           "unit": "Bps"
2238         },
2239         "overrides": []
2240       },
2241       "gridPos": {
2242         "h": 8,
2243         "w": 8,
2244         "x": 16,
2245         "y": 28
2246       },
2247       "id": 6,
2248       "links": [],
2249       "options": {
2250         "legend": {
2251           "calcs": [
2252             "mean",
2253             "lastNotNull",
2254             "max"
2255           ],
2256           "displayMode": "table",
2257           "placement": "bottom",
2258           "showLegend": true
2259         },
2260         "tooltip": {
2261           "mode": "single",
2262           "sort": "none"
2263         }
2264       },
2265       "pluginVersion": "8.1.3",
2266       "targets": [
2267         {
2268           "datasource": {
2269             "type": "prometheus",
2270             "uid": "PBFA97CFB590B2093"
2271           },
2272           "exemplar": true,
2273           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2274           "interval": "",
2275           "intervalFactor": 2,
2276           "legendFormat": "bytes/sec",
2277           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2278           "refId": "A",
2279           "step": 4
2280         }
2281       ],
2282       "title": "Bytes Out",
2283       "type": "timeseries"
2284     },
2285     {
2286       "datasource": {
2287         "type": "prometheus",
2288         "uid": "PBFA97CFB590B2093"
2289       },
2290       "fieldConfig": {
2291         "defaults": {
2292           "color": {
2293             "mode": "palette-classic"
2294           },
2295           "custom": {
2296             "axisCenteredZero": false,
2297             "axisColorMode": "text",
2298             "axisLabel": "Messages/s",
2299             "axisPlacement": "auto",
2300             "barAlignment": 0,
2301             "drawStyle": "line",
2302             "fillOpacity": 10,
2303             "gradientMode": "none",
2304             "hideFrom": {
2305               "legend": false,
2306               "tooltip": false,
2307               "viz": false
2308             },
2309             "lineInterpolation": "linear",
2310             "lineWidth": 2,
2311             "pointSize": 5,
2312             "scaleDistribution": {
2313               "type": "linear"
2314             },
2315             "showPoints": "never",
2316             "spanNulls": false,
2317             "stacking": {
2318               "group": "A",
2319               "mode": "none"
2320             },
2321             "thresholdsStyle": {
2322               "mode": "off"
2323             }
2324           },
2325           "links": [],
2326           "mappings": [],
2327           "thresholds": {
2328             "mode": "absolute",
2329             "steps": [
2330               {
2331                 "color": "green",
2332                 "value": null
2333               },
2334               {
2335                 "color": "red",
2336                 "value": 80
2337               }
2338             ]
2339           },
2340           "unit": "cps"
2341         },
2342         "overrides": []
2343       },
2344       "gridPos": {
2345         "h": 8,
2346         "w": 8,
2347         "x": 0,
2348         "y": 36
2349       },
2350       "id": 10,
2351       "links": [],
2352       "options": {
2353         "legend": {
2354           "calcs": [
2355             "mean",
2356             "lastNotNull",
2357             "max"
2358           ],
2359           "displayMode": "table",
2360           "placement": "bottom",
2361           "showLegend": true
2362         },
2363         "tooltip": {
2364           "mode": "single",
2365           "sort": "none"
2366         }
2367       },
2368       "pluginVersion": "8.1.3",
2369       "targets": [
2370         {
2371           "datasource": {
2372             "type": "prometheus",
2373             "uid": "PBFA97CFB590B2093"
2374           },
2375           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2376           "format": "time_series",
2377           "interval": "",
2378           "intervalFactor": 2,
2379           "legendFormat": "{{instance}}",
2380           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2381           "refId": "A",
2382           "step": 4
2383         }
2384       ],
2385       "title": "Messages In Per Broker",
2386       "type": "timeseries"
2387     },
2388     {
2389       "datasource": {
2390         "type": "prometheus",
2391         "uid": "PBFA97CFB590B2093"
2392       },
2393       "fieldConfig": {
2394         "defaults": {
2395           "color": {
2396             "mode": "palette-classic"
2397           },
2398           "custom": {
2399             "axisCenteredZero": false,
2400             "axisColorMode": "text",
2401             "axisLabel": "Bytes/s",
2402             "axisPlacement": "auto",
2403             "barAlignment": 0,
2404             "drawStyle": "line",
2405             "fillOpacity": 10,
2406             "gradientMode": "none",
2407             "hideFrom": {
2408               "legend": false,
2409               "tooltip": false,
2410               "viz": false
2411             },
2412             "lineInterpolation": "linear",
2413             "lineWidth": 2,
2414             "pointSize": 5,
2415             "scaleDistribution": {
2416               "type": "linear"
2417             },
2418             "showPoints": "never",
2419             "spanNulls": false,
2420             "stacking": {
2421               "group": "A",
2422               "mode": "none"
2423             },
2424             "thresholdsStyle": {
2425               "mode": "off"
2426             }
2427           },
2428           "links": [],
2429           "mappings": [],
2430           "thresholds": {
2431             "mode": "absolute",
2432             "steps": [
2433               {
2434                 "color": "green",
2435                 "value": null
2436               },
2437               {
2438                 "color": "red",
2439                 "value": 80
2440               }
2441             ]
2442           },
2443           "unit": "Bps"
2444         },
2445         "overrides": []
2446       },
2447       "gridPos": {
2448         "h": 8,
2449         "w": 8,
2450         "x": 8,
2451         "y": 36
2452       },
2453       "id": 7,
2454       "links": [],
2455       "options": {
2456         "legend": {
2457           "calcs": [
2458             "mean",
2459             "lastNotNull",
2460             "max"
2461           ],
2462           "displayMode": "table",
2463           "placement": "bottom",
2464           "showLegend": true
2465         },
2466         "tooltip": {
2467           "mode": "single",
2468           "sort": "none"
2469         }
2470       },
2471       "pluginVersion": "8.1.3",
2472       "targets": [
2473         {
2474           "datasource": {
2475             "type": "prometheus",
2476             "uid": "PBFA97CFB590B2093"
2477           },
2478           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2479           "format": "time_series",
2480           "interval": "",
2481           "intervalFactor": 2,
2482           "legendFormat": "{{instance}}",
2483           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2484           "refId": "A",
2485           "step": 4
2486         }
2487       ],
2488       "title": "Bytes In Per Broker",
2489       "type": "timeseries"
2490     },
2491     {
2492       "datasource": {
2493         "type": "prometheus",
2494         "uid": "PBFA97CFB590B2093"
2495       },
2496       "fieldConfig": {
2497         "defaults": {
2498           "color": {
2499             "mode": "palette-classic"
2500           },
2501           "custom": {
2502             "axisCenteredZero": false,
2503             "axisColorMode": "text",
2504             "axisLabel": "",
2505             "axisPlacement": "auto",
2506             "barAlignment": 0,
2507             "drawStyle": "line",
2508             "fillOpacity": 10,
2509             "gradientMode": "none",
2510             "hideFrom": {
2511               "legend": false,
2512               "tooltip": false,
2513               "viz": false
2514             },
2515             "lineInterpolation": "linear",
2516             "lineWidth": 1,
2517             "pointSize": 5,
2518             "scaleDistribution": {
2519               "type": "linear"
2520             },
2521             "showPoints": "never",
2522             "spanNulls": true,
2523             "stacking": {
2524               "group": "A",
2525               "mode": "none"
2526             },
2527             "thresholdsStyle": {
2528               "mode": "off"
2529             }
2530           },
2531           "links": [],
2532           "mappings": [],
2533           "min": 0,
2534           "thresholds": {
2535             "mode": "absolute",
2536             "steps": [
2537               {
2538                 "color": "green",
2539                 "value": null
2540               },
2541               {
2542                 "color": "red",
2543                 "value": 80
2544               }
2545             ]
2546           },
2547           "unit": "binBps"
2548         },
2549         "overrides": []
2550       },
2551       "gridPos": {
2552         "h": 8,
2553         "w": 8,
2554         "x": 16,
2555         "y": 36
2556       },
2557       "id": 9,
2558       "links": [],
2559       "options": {
2560         "legend": {
2561           "calcs": [
2562             "mean",
2563             "lastNotNull",
2564             "max"
2565           ],
2566           "displayMode": "table",
2567           "placement": "bottom",
2568           "showLegend": true
2569         },
2570         "tooltip": {
2571           "mode": "single",
2572           "sort": "none"
2573         }
2574       },
2575       "pluginVersion": "8.1.3",
2576       "targets": [
2577         {
2578           "datasource": {
2579             "type": "prometheus",
2580             "uid": "PBFA97CFB590B2093"
2581           },
2582           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2583           "format": "time_series",
2584           "interval": "",
2585           "intervalFactor": 1,
2586           "legendFormat": "{{instance}}",
2587           "refId": "A"
2588         }
2589       ],
2590       "title": "Bytes Out Per Broker",
2591       "type": "timeseries"
2592     },
2593     {
2594       "collapsed": false,
2595       "datasource": {
2596         "type": "datasource",
2597         "uid": "grafana"
2598       },
2599       "gridPos": {
2600         "h": 1,
2601         "w": 24,
2602         "x": 0,
2603         "y": 44
2604       },
2605       "id": 44,
2606       "panels": [],
2607       "targets": [
2608         {
2609           "datasource": {
2610             "type": "datasource",
2611             "uid": "grafana"
2612           },
2613           "refId": "A"
2614         }
2615       ],
2616       "title": "Thread utilization",
2617       "type": "row"
2618     },
2619     {
2620       "datasource": {
2621         "type": "prometheus",
2622         "uid": "PBFA97CFB590B2093"
2623       },
2624       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2625       "fieldConfig": {
2626         "defaults": {
2627           "color": {
2628             "mode": "palette-classic"
2629           },
2630           "custom": {
2631             "axisCenteredZero": false,
2632             "axisColorMode": "text",
2633             "axisLabel": "",
2634             "axisPlacement": "auto",
2635             "barAlignment": 0,
2636             "drawStyle": "line",
2637             "fillOpacity": 10,
2638             "gradientMode": "none",
2639             "hideFrom": {
2640               "legend": false,
2641               "tooltip": false,
2642               "viz": false
2643             },
2644             "lineInterpolation": "linear",
2645             "lineWidth": 1,
2646             "pointSize": 5,
2647             "scaleDistribution": {
2648               "type": "linear"
2649             },
2650             "showPoints": "never",
2651             "spanNulls": true,
2652             "stacking": {
2653               "group": "A",
2654               "mode": "none"
2655             },
2656             "thresholdsStyle": {
2657               "mode": "off"
2658             }
2659           },
2660           "links": [],
2661           "mappings": [],
2662           "thresholds": {
2663             "mode": "absolute",
2664             "steps": [
2665               {
2666                 "color": "green"
2667               },
2668               {
2669                 "color": "red",
2670                 "value": 80
2671               }
2672             ]
2673           },
2674           "unit": "percentunit"
2675         },
2676         "overrides": []
2677       },
2678       "gridPos": {
2679         "h": 12,
2680         "w": 12,
2681         "x": 0,
2682         "y": 45
2683       },
2684       "id": 24,
2685       "options": {
2686         "legend": {
2687           "calcs": [
2688             "mean",
2689             "lastNotNull",
2690             "max"
2691           ],
2692           "displayMode": "table",
2693           "placement": "bottom",
2694           "showLegend": true
2695         },
2696         "tooltip": {
2697           "mode": "single",
2698           "sort": "none"
2699         }
2700       },
2701       "pluginVersion": "8.1.3",
2702       "targets": [
2703         {
2704           "datasource": {
2705             "type": "prometheus",
2706             "uid": "PBFA97CFB590B2093"
2707           },
2708           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2709           "interval": "",
2710           "legendFormat": "{{instance}}",
2711           "refId": "A"
2712         }
2713       ],
2714       "title": "Network Processor Avg Usage Percent",
2715       "type": "timeseries"
2716     },
2717     {
2718       "datasource": {
2719         "type": "prometheus",
2720         "uid": "PBFA97CFB590B2093"
2721       },
2722       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2723       "fieldConfig": {
2724         "defaults": {
2725           "color": {
2726             "mode": "palette-classic"
2727           },
2728           "custom": {
2729             "axisCenteredZero": false,
2730             "axisColorMode": "text",
2731             "axisLabel": "",
2732             "axisPlacement": "auto",
2733             "barAlignment": 0,
2734             "drawStyle": "line",
2735             "fillOpacity": 10,
2736             "gradientMode": "none",
2737             "hideFrom": {
2738               "legend": false,
2739               "tooltip": false,
2740               "viz": false
2741             },
2742             "lineInterpolation": "linear",
2743             "lineWidth": 1,
2744             "pointSize": 5,
2745             "scaleDistribution": {
2746               "type": "linear"
2747             },
2748             "showPoints": "never",
2749             "spanNulls": true,
2750             "stacking": {
2751               "group": "A",
2752               "mode": "none"
2753             },
2754             "thresholdsStyle": {
2755               "mode": "off"
2756             }
2757           },
2758           "links": [],
2759           "mappings": [],
2760           "min": 0,
2761           "thresholds": {
2762             "mode": "absolute",
2763             "steps": [
2764               {
2765                 "color": "green"
2766               },
2767               {
2768                 "color": "red",
2769                 "value": 80
2770               }
2771             ]
2772           },
2773           "unit": "percentunit"
2774         },
2775         "overrides": []
2776       },
2777       "gridPos": {
2778         "h": 12,
2779         "w": 12,
2780         "x": 12,
2781         "y": 45
2782       },
2783       "id": 25,
2784       "options": {
2785         "legend": {
2786           "calcs": [
2787             "mean",
2788             "lastNotNull",
2789             "max"
2790           ],
2791           "displayMode": "table",
2792           "placement": "bottom",
2793           "showLegend": true
2794         },
2795         "tooltip": {
2796           "mode": "single",
2797           "sort": "none"
2798         }
2799       },
2800       "pluginVersion": "8.1.3",
2801       "targets": [
2802         {
2803           "datasource": {
2804             "type": "prometheus",
2805             "uid": "PBFA97CFB590B2093"
2806           },
2807           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2808           "interval": "",
2809           "legendFormat": "{{instance}}",
2810           "refId": "A"
2811         }
2812       ],
2813       "title": "Request Handler Avg Percent",
2814       "type": "timeseries"
2815     },
2816     {
2817       "collapsed": false,
2818       "datasource": {
2819         "type": "datasource",
2820         "uid": "grafana"
2821       },
2822       "gridPos": {
2823         "h": 1,
2824         "w": 24,
2825         "x": 0,
2826         "y": 57
2827       },
2828       "id": 82,
2829       "panels": [],
2830       "targets": [
2831         {
2832           "datasource": {
2833             "type": "datasource",
2834             "uid": "grafana"
2835           },
2836           "refId": "A"
2837         }
2838       ],
2839       "title": "Isr Shrinks / Expands",
2840       "type": "row"
2841     },
2842     {
2843       "datasource": {
2844         "type": "prometheus",
2845         "uid": "PBFA97CFB590B2093"
2846       },
2847       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2848       "fieldConfig": {
2849         "defaults": {
2850           "color": {
2851             "mode": "palette-classic"
2852           },
2853           "custom": {
2854             "axisCenteredZero": false,
2855             "axisColorMode": "text",
2856             "axisLabel": "",
2857             "axisPlacement": "auto",
2858             "barAlignment": 0,
2859             "drawStyle": "line",
2860             "fillOpacity": 10,
2861             "gradientMode": "none",
2862             "hideFrom": {
2863               "legend": false,
2864               "tooltip": false,
2865               "viz": false
2866             },
2867             "lineInterpolation": "linear",
2868             "lineWidth": 1,
2869             "pointSize": 5,
2870             "scaleDistribution": {
2871               "type": "linear"
2872             },
2873             "showPoints": "never",
2874             "spanNulls": true,
2875             "stacking": {
2876               "group": "A",
2877               "mode": "none"
2878             },
2879             "thresholdsStyle": {
2880               "mode": "off"
2881             }
2882           },
2883           "links": [],
2884           "mappings": [],
2885           "min": 0,
2886           "thresholds": {
2887             "mode": "absolute",
2888             "steps": [
2889               {
2890                 "color": "green"
2891               },
2892               {
2893                 "color": "red",
2894                 "value": 80
2895               }
2896             ]
2897           },
2898           "unit": "ops"
2899         },
2900         "overrides": [
2901           {
2902             "__systemRef": "hideSeriesFrom",
2903             "matcher": {
2904               "id": "byNames",
2905               "options": {
2906                 "mode": "exclude",
2907                 "names": [
2908                   "kafka-2:80"
2909                 ],
2910                 "prefix": "All except:",
2911                 "readOnly": true
2912               }
2913             },
2914             "properties": [
2915               {
2916                 "id": "custom.hideFrom",
2917                 "value": {
2918                   "legend": false,
2919                   "tooltip": false,
2920                   "viz": true
2921                 }
2922               }
2923             ]
2924           }
2925         ]
2926       },
2927       "gridPos": {
2928         "h": 9,
2929         "w": 12,
2930         "x": 0,
2931         "y": 58
2932       },
2933       "id": 80,
2934       "options": {
2935         "legend": {
2936           "calcs": [
2937             "mean",
2938             "lastNotNull",
2939             "max"
2940           ],
2941           "displayMode": "table",
2942           "placement": "bottom",
2943           "showLegend": true
2944         },
2945         "tooltip": {
2946           "mode": "single",
2947           "sort": "none"
2948         }
2949       },
2950       "pluginVersion": "8.1.3",
2951       "targets": [
2952         {
2953           "datasource": {
2954             "type": "prometheus",
2955             "uid": "PBFA97CFB590B2093"
2956           },
2957           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2958           "interval": "",
2959           "legendFormat": "{{instance}}",
2960           "refId": "A"
2961         }
2962       ],
2963       "title": "IsrShrinks per Sec",
2964       "type": "timeseries"
2965     },
2966     {
2967       "datasource": {
2968         "type": "prometheus",
2969         "uid": "PBFA97CFB590B2093"
2970       },
2971       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2972       "fieldConfig": {
2973         "defaults": {
2974           "color": {
2975             "mode": "palette-classic"
2976           },
2977           "custom": {
2978             "axisCenteredZero": false,
2979             "axisColorMode": "text",
2980             "axisLabel": "",
2981             "axisPlacement": "auto",
2982             "barAlignment": 0,
2983             "drawStyle": "line",
2984             "fillOpacity": 10,
2985             "gradientMode": "none",
2986             "hideFrom": {
2987               "legend": false,
2988               "tooltip": false,
2989               "viz": false
2990             },
2991             "lineInterpolation": "linear",
2992             "lineWidth": 1,
2993             "pointSize": 5,
2994             "scaleDistribution": {
2995               "type": "linear"
2996             },
2997             "showPoints": "never",
2998             "spanNulls": true,
2999             "stacking": {
3000               "group": "A",
3001               "mode": "none"
3002             },
3003             "thresholdsStyle": {
3004               "mode": "off"
3005             }
3006           },
3007           "links": [],
3008           "mappings": [],
3009           "min": 0,
3010           "thresholds": {
3011             "mode": "absolute",
3012             "steps": [
3013               {
3014                 "color": "green"
3015               },
3016               {
3017                 "color": "red",
3018                 "value": 80
3019               }
3020             ]
3021           },
3022           "unit": "ops"
3023         },
3024         "overrides": [
3025           {
3026             "__systemRef": "hideSeriesFrom",
3027             "matcher": {
3028               "id": "byNames",
3029               "options": {
3030                 "mode": "exclude",
3031                 "names": [
3032                   "kafka-2:80"
3033                 ],
3034                 "prefix": "All except:",
3035                 "readOnly": true
3036               }
3037             },
3038             "properties": [
3039               {
3040                 "id": "custom.hideFrom",
3041                 "value": {
3042                   "legend": false,
3043                   "tooltip": false,
3044                   "viz": true
3045                 }
3046               }
3047             ]
3048           }
3049         ]
3050       },
3051       "gridPos": {
3052         "h": 9,
3053         "w": 12,
3054         "x": 12,
3055         "y": 58
3056       },
3057       "id": 83,
3058       "options": {
3059         "legend": {
3060           "calcs": [
3061             "mean",
3062             "lastNotNull",
3063             "max"
3064           ],
3065           "displayMode": "table",
3066           "placement": "bottom",
3067           "showLegend": true
3068         },
3069         "tooltip": {
3070           "mode": "single",
3071           "sort": "none"
3072         }
3073       },
3074       "pluginVersion": "8.1.3",
3075       "targets": [
3076         {
3077           "datasource": {
3078             "type": "prometheus",
3079             "uid": "PBFA97CFB590B2093"
3080           },
3081           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3082           "hide": false,
3083           "interval": "",
3084           "legendFormat": "{{instance}}",
3085           "refId": "A"
3086         }
3087       ],
3088       "title": "IsrExpands per Sec",
3089       "type": "timeseries"
3090     },
3091     {
3092       "collapsed": true,
3093       "datasource": {
3094         "type": "datasource",
3095         "uid": "grafana"
3096       },
3097       "gridPos": {
3098         "h": 1,
3099         "w": 24,
3100         "x": 0,
3101         "y": 67
3102       },
3103       "id": 53,
3104       "panels": [
3105         {
3106           "datasource": {
3107             "type": "prometheus",
3108             "uid": "PBFA97CFB590B2093"
3109           },
3110           "fieldConfig": {
3111             "defaults": {
3112               "color": {
3113                 "mode": "palette-classic"
3114               },
3115               "custom": {
3116                 "axisLabel": "",
3117                 "axisPlacement": "auto",
3118                 "barAlignment": 0,
3119                 "drawStyle": "line",
3120                 "fillOpacity": 10,
3121                 "gradientMode": "none",
3122                 "hideFrom": {
3123                   "legend": false,
3124                   "tooltip": false,
3125                   "viz": false
3126                 },
3127                 "lineInterpolation": "linear",
3128                 "lineWidth": 1,
3129                 "pointSize": 5,
3130                 "scaleDistribution": {
3131                   "type": "linear"
3132                 },
3133                 "showPoints": "never",
3134                 "spanNulls": true,
3135                 "stacking": {
3136                   "group": "A",
3137                   "mode": "none"
3138                 },
3139                 "thresholdsStyle": {
3140                   "mode": "off"
3141                 }
3142               },
3143               "links": [],
3144               "mappings": [],
3145               "thresholds": {
3146                 "mode": "absolute",
3147                 "steps": [
3148                   {
3149                     "color": "green"
3150                   },
3151                   {
3152                     "color": "red",
3153                     "value": 80
3154                   }
3155                 ]
3156               },
3157               "unit": "bytes"
3158             },
3159             "overrides": []
3160           },
3161           "gridPos": {
3162             "h": 9,
3163             "w": 12,
3164             "x": 0,
3165             "y": 28
3166           },
3167           "id": 55,
3168           "options": {
3169             "legend": {
3170               "calcs": [
3171                 "mean",
3172                 "lastNotNull",
3173                 "max"
3174               ],
3175               "displayMode": "table",
3176               "placement": "bottom",
3177               "showLegend": true
3178             },
3179             "tooltip": {
3180               "mode": "single"
3181             }
3182           },
3183           "pluginVersion": "8.1.3",
3184           "targets": [
3185             {
3186               "datasource": {
3187                 "type": "prometheus",
3188                 "uid": "PBFA97CFB590B2093"
3189               },
3190               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3191               "legendFormat": "{{topic}}",
3192               "refId": "A"
3193             }
3194           ],
3195           "title": "Log size per Topic",
3196           "type": "timeseries"
3197         },
3198         {
3199           "datasource": {
3200             "type": "prometheus",
3201             "uid": "PBFA97CFB590B2093"
3202           },
3203           "fieldConfig": {
3204             "defaults": {
3205               "color": {
3206                 "mode": "palette-classic"
3207               },
3208               "custom": {
3209                 "axisLabel": "",
3210                 "axisPlacement": "auto",
3211                 "barAlignment": 0,
3212                 "drawStyle": "line",
3213                 "fillOpacity": 10,
3214                 "gradientMode": "none",
3215                 "hideFrom": {
3216                   "legend": false,
3217                   "tooltip": false,
3218                   "viz": false
3219                 },
3220                 "lineInterpolation": "linear",
3221                 "lineWidth": 1,
3222                 "pointSize": 5,
3223                 "scaleDistribution": {
3224                   "type": "linear"
3225                 },
3226                 "showPoints": "never",
3227                 "spanNulls": true,
3228                 "stacking": {
3229                   "group": "A",
3230                   "mode": "none"
3231                 },
3232                 "thresholdsStyle": {
3233                   "mode": "off"
3234                 }
3235               },
3236               "links": [],
3237               "mappings": [],
3238               "thresholds": {
3239                 "mode": "absolute",
3240                 "steps": [
3241                   {
3242                     "color": "green"
3243                   },
3244                   {
3245                     "color": "red",
3246                     "value": 80
3247                   }
3248                 ]
3249               },
3250               "unit": "bytes"
3251             },
3252             "overrides": []
3253           },
3254           "gridPos": {
3255             "h": 9,
3256             "w": 12,
3257             "x": 12,
3258             "y": 28
3259           },
3260           "id": 56,
3261           "options": {
3262             "legend": {
3263               "calcs": [
3264                 "mean",
3265                 "lastNotNull",
3266                 "max"
3267               ],
3268               "displayMode": "table",
3269               "placement": "bottom",
3270               "showLegend": true
3271             },
3272             "tooltip": {
3273               "mode": "single"
3274             }
3275           },
3276           "pluginVersion": "8.1.3",
3277           "targets": [
3278             {
3279               "datasource": {
3280                 "type": "prometheus",
3281                 "uid": "PBFA97CFB590B2093"
3282               },
3283               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3284               "legendFormat": "{{instance}}",
3285               "refId": "A"
3286             }
3287           ],
3288           "title": "Log size per Broker",
3289           "type": "timeseries"
3290         }
3291       ],
3292       "targets": [
3293         {
3294           "datasource": {
3295             "type": "datasource",
3296             "uid": "grafana"
3297           },
3298           "refId": "A"
3299         }
3300       ],
3301       "title": "Logs size",
3302       "type": "row"
3303     },
3304     {
3305       "collapsed": true,
3306       "datasource": {
3307         "type": "datasource",
3308         "uid": "grafana"
3309       },
3310       "gridPos": {
3311         "h": 1,
3312         "w": 24,
3313         "x": 0,
3314         "y": 68
3315       },
3316       "id": 58,
3317       "panels": [
3318         {
3319           "datasource": {
3320             "type": "prometheus",
3321             "uid": "PBFA97CFB590B2093"
3322           },
3323           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3324           "fieldConfig": {
3325             "defaults": {
3326               "color": {
3327                 "mode": "palette-classic"
3328               },
3329               "custom": {
3330                 "axisLabel": "",
3331                 "axisPlacement": "auto",
3332                 "barAlignment": 0,
3333                 "drawStyle": "line",
3334                 "fillOpacity": 10,
3335                 "gradientMode": "none",
3336                 "hideFrom": {
3337                   "legend": false,
3338                   "tooltip": false,
3339                   "viz": false
3340                 },
3341                 "lineInterpolation": "linear",
3342                 "lineWidth": 1,
3343                 "pointSize": 5,
3344                 "scaleDistribution": {
3345                   "type": "linear"
3346                 },
3347                 "showPoints": "never",
3348                 "spanNulls": true,
3349                 "stacking": {
3350                   "group": "A",
3351                   "mode": "none"
3352                 },
3353                 "thresholdsStyle": {
3354                   "mode": "off"
3355                 }
3356               },
3357               "links": [],
3358               "mappings": [],
3359               "thresholds": {
3360                 "mode": "absolute",
3361                 "steps": [
3362                   {
3363                     "color": "green"
3364                   },
3365                   {
3366                     "color": "red",
3367                     "value": 80
3368                   }
3369                 ]
3370               },
3371               "unit": "ms"
3372             },
3373             "overrides": []
3374           },
3375           "gridPos": {
3376             "h": 9,
3377             "w": 12,
3378             "x": 0,
3379             "y": 29
3380           },
3381           "id": 60,
3382           "options": {
3383             "legend": {
3384               "calcs": [
3385                 "mean",
3386                 "lastNotNull",
3387                 "max"
3388               ],
3389               "displayMode": "list",
3390               "placement": "bottom",
3391               "showLegend": true
3392             },
3393             "tooltip": {
3394               "mode": "single"
3395             }
3396           },
3397           "pluginVersion": "8.1.3",
3398           "targets": [
3399             {
3400               "datasource": {
3401                 "type": "prometheus",
3402                 "uid": "PBFA97CFB590B2093"
3403               },
3404               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3405               "hide": false,
3406               "legendFormat": "{{instance}} - {{quantile}}",
3407               "refId": "A"
3408             }
3409           ],
3410           "title": "Producer - RequestQueueTimeMs",
3411           "type": "timeseries"
3412         },
3413         {
3414           "datasource": {
3415             "type": "prometheus",
3416             "uid": "PBFA97CFB590B2093"
3417           },
3418           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3419           "fieldConfig": {
3420             "defaults": {
3421               "color": {
3422                 "mode": "palette-classic"
3423               },
3424               "custom": {
3425                 "axisLabel": "",
3426                 "axisPlacement": "auto",
3427                 "barAlignment": 0,
3428                 "drawStyle": "line",
3429                 "fillOpacity": 10,
3430                 "gradientMode": "none",
3431                 "hideFrom": {
3432                   "legend": false,
3433                   "tooltip": false,
3434                   "viz": false
3435                 },
3436                 "lineInterpolation": "linear",
3437                 "lineWidth": 1,
3438                 "pointSize": 5,
3439                 "scaleDistribution": {
3440                   "type": "linear"
3441                 },
3442                 "showPoints": "never",
3443                 "spanNulls": true,
3444                 "stacking": {
3445                   "group": "A",
3446                   "mode": "none"
3447                 },
3448                 "thresholdsStyle": {
3449                   "mode": "off"
3450                 }
3451               },
3452               "links": [],
3453               "mappings": [],
3454               "thresholds": {
3455                 "mode": "absolute",
3456                 "steps": [
3457                   {
3458                     "color": "green"
3459                   },
3460                   {
3461                     "color": "red",
3462                     "value": 80
3463                   }
3464                 ]
3465               },
3466               "unit": "ms"
3467             },
3468             "overrides": []
3469           },
3470           "gridPos": {
3471             "h": 9,
3472             "w": 12,
3473             "x": 12,
3474             "y": 29
3475           },
3476           "id": 61,
3477           "options": {
3478             "legend": {
3479               "calcs": [
3480                 "mean",
3481                 "lastNotNull",
3482                 "max"
3483               ],
3484               "displayMode": "list",
3485               "placement": "bottom",
3486               "showLegend": true
3487             },
3488             "tooltip": {
3489               "mode": "single"
3490             }
3491           },
3492           "pluginVersion": "8.1.3",
3493           "targets": [
3494             {
3495               "datasource": {
3496                 "type": "prometheus",
3497                 "uid": "PBFA97CFB590B2093"
3498               },
3499               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3500               "hide": false,
3501               "legendFormat": "{{instance}} - {{quantile}}",
3502               "refId": "A"
3503             }
3504           ],
3505           "title": "Producer - LocalTimeMs",
3506           "type": "timeseries"
3507         },
3508         {
3509           "datasource": {
3510             "type": "prometheus",
3511             "uid": "PBFA97CFB590B2093"
3512           },
3513           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3514           "fieldConfig": {
3515             "defaults": {
3516               "color": {
3517                 "mode": "palette-classic"
3518               },
3519               "custom": {
3520                 "axisLabel": "",
3521                 "axisPlacement": "auto",
3522                 "barAlignment": 0,
3523                 "drawStyle": "line",
3524                 "fillOpacity": 10,
3525                 "gradientMode": "none",
3526                 "hideFrom": {
3527                   "legend": false,
3528                   "tooltip": false,
3529                   "viz": false
3530                 },
3531                 "lineInterpolation": "linear",
3532                 "lineWidth": 1,
3533                 "pointSize": 5,
3534                 "scaleDistribution": {
3535                   "type": "linear"
3536                 },
3537                 "showPoints": "never",
3538                 "spanNulls": true,
3539                 "stacking": {
3540                   "group": "A",
3541                   "mode": "none"
3542                 },
3543                 "thresholdsStyle": {
3544                   "mode": "off"
3545                 }
3546               },
3547               "links": [],
3548               "mappings": [],
3549               "thresholds": {
3550                 "mode": "absolute",
3551                 "steps": [
3552                   {
3553                     "color": "green"
3554                   },
3555                   {
3556                     "color": "red",
3557                     "value": 80
3558                   }
3559                 ]
3560               },
3561               "unit": "ms"
3562             },
3563             "overrides": []
3564           },
3565           "gridPos": {
3566             "h": 10,
3567             "w": 8,
3568             "x": 0,
3569             "y": 38
3570           },
3571           "id": 62,
3572           "options": {
3573             "legend": {
3574               "calcs": [
3575                 "mean",
3576                 "lastNotNull",
3577                 "max"
3578               ],
3579               "displayMode": "table",
3580               "placement": "bottom",
3581               "showLegend": true
3582             },
3583             "tooltip": {
3584               "mode": "single"
3585             }
3586           },
3587           "pluginVersion": "8.1.3",
3588           "targets": [
3589             {
3590               "datasource": {
3591                 "type": "prometheus",
3592                 "uid": "PBFA97CFB590B2093"
3593               },
3594               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3595               "hide": false,
3596               "legendFormat": "{{instance}} - {{quantile}}",
3597               "refId": "A"
3598             }
3599           ],
3600           "title": "Producer - RemoteTimeMs",
3601           "type": "timeseries"
3602         },
3603         {
3604           "datasource": {
3605             "type": "prometheus",
3606             "uid": "PBFA97CFB590B2093"
3607           },
3608           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3609           "fieldConfig": {
3610             "defaults": {
3611               "color": {
3612                 "mode": "palette-classic"
3613               },
3614               "custom": {
3615                 "axisLabel": "",
3616                 "axisPlacement": "auto",
3617                 "barAlignment": 0,
3618                 "drawStyle": "line",
3619                 "fillOpacity": 10,
3620                 "gradientMode": "none",
3621                 "hideFrom": {
3622                   "legend": false,
3623                   "tooltip": false,
3624                   "viz": false
3625                 },
3626                 "lineInterpolation": "linear",
3627                 "lineWidth": 1,
3628                 "pointSize": 5,
3629                 "scaleDistribution": {
3630                   "type": "linear"
3631                 },
3632                 "showPoints": "never",
3633                 "spanNulls": true,
3634                 "stacking": {
3635                   "group": "A",
3636                   "mode": "none"
3637                 },
3638                 "thresholdsStyle": {
3639                   "mode": "off"
3640                 }
3641               },
3642               "links": [],
3643               "mappings": [],
3644               "thresholds": {
3645                 "mode": "absolute",
3646                 "steps": [
3647                   {
3648                     "color": "green"
3649                   },
3650                   {
3651                     "color": "red",
3652                     "value": 80
3653                   }
3654                 ]
3655               },
3656               "unit": "ms"
3657             },
3658             "overrides": []
3659           },
3660           "gridPos": {
3661             "h": 10,
3662             "w": 8,
3663             "x": 8,
3664             "y": 38
3665           },
3666           "id": 63,
3667           "options": {
3668             "legend": {
3669               "calcs": [
3670                 "mean",
3671                 "lastNotNull",
3672                 "max"
3673               ],
3674               "displayMode": "table",
3675               "placement": "bottom",
3676               "showLegend": true
3677             },
3678             "tooltip": {
3679               "mode": "single"
3680             }
3681           },
3682           "pluginVersion": "8.1.3",
3683           "targets": [
3684             {
3685               "datasource": {
3686                 "type": "prometheus",
3687                 "uid": "PBFA97CFB590B2093"
3688               },
3689               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3690               "hide": false,
3691               "legendFormat": "{{instance}} - {{quantile}}",
3692               "refId": "A"
3693             }
3694           ],
3695           "title": "Producer - ResponseQueueTimeMs",
3696           "type": "timeseries"
3697         },
3698         {
3699           "datasource": {
3700             "type": "prometheus",
3701             "uid": "PBFA97CFB590B2093"
3702           },
3703           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3704           "fieldConfig": {
3705             "defaults": {
3706               "color": {
3707                 "mode": "palette-classic"
3708               },
3709               "custom": {
3710                 "axisLabel": "",
3711                 "axisPlacement": "auto",
3712                 "barAlignment": 0,
3713                 "drawStyle": "line",
3714                 "fillOpacity": 10,
3715                 "gradientMode": "none",
3716                 "hideFrom": {
3717                   "legend": false,
3718                   "tooltip": false,
3719                   "viz": false
3720                 },
3721                 "lineInterpolation": "linear",
3722                 "lineWidth": 1,
3723                 "pointSize": 5,
3724                 "scaleDistribution": {
3725                   "type": "linear"
3726                 },
3727                 "showPoints": "never",
3728                 "spanNulls": true,
3729                 "stacking": {
3730                   "group": "A",
3731                   "mode": "none"
3732                 },
3733                 "thresholdsStyle": {
3734                   "mode": "off"
3735                 }
3736               },
3737               "links": [],
3738               "mappings": [],
3739               "min": 0,
3740               "thresholds": {
3741                 "mode": "absolute",
3742                 "steps": [
3743                   {
3744                     "color": "green"
3745                   },
3746                   {
3747                     "color": "red",
3748                     "value": 80
3749                   }
3750                 ]
3751               },
3752               "unit": "ms"
3753             },
3754             "overrides": []
3755           },
3756           "gridPos": {
3757             "h": 10,
3758             "w": 8,
3759             "x": 16,
3760             "y": 38
3761           },
3762           "id": 64,
3763           "options": {
3764             "legend": {
3765               "calcs": [
3766                 "mean",
3767                 "lastNotNull",
3768                 "max"
3769               ],
3770               "displayMode": "table",
3771               "placement": "bottom",
3772               "showLegend": true
3773             },
3774             "tooltip": {
3775               "mode": "single"
3776             }
3777           },
3778           "pluginVersion": "8.1.3",
3779           "targets": [
3780             {
3781               "datasource": {
3782                 "type": "prometheus",
3783                 "uid": "PBFA97CFB590B2093"
3784               },
3785               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3786               "hide": false,
3787               "legendFormat": "{{instance}} - {{quantile}}",
3788               "refId": "A"
3789             }
3790           ],
3791           "title": "Producer - ResponseSendTimeMs",
3792           "type": "timeseries"
3793         }
3794       ],
3795       "targets": [
3796         {
3797           "datasource": {
3798             "type": "datasource",
3799             "uid": "grafana"
3800           },
3801           "refId": "A"
3802         }
3803       ],
3804       "title": "Producer Performance",
3805       "type": "row"
3806     },
3807     {
3808       "collapsed": true,
3809       "datasource": {
3810         "type": "datasource",
3811         "uid": "grafana"
3812       },
3813       "gridPos": {
3814         "h": 1,
3815         "w": 24,
3816         "x": 0,
3817         "y": 69
3818       },
3819       "id": 68,
3820       "panels": [
3821         {
3822           "datasource": {
3823             "type": "prometheus",
3824             "uid": "PBFA97CFB590B2093"
3825           },
3826           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3827           "fieldConfig": {
3828             "defaults": {
3829               "color": {
3830                 "mode": "palette-classic"
3831               },
3832               "custom": {
3833                 "axisLabel": "",
3834                 "axisPlacement": "auto",
3835                 "barAlignment": 0,
3836                 "drawStyle": "line",
3837                 "fillOpacity": 10,
3838                 "gradientMode": "none",
3839                 "hideFrom": {
3840                   "legend": false,
3841                   "tooltip": false,
3842                   "viz": false
3843                 },
3844                 "lineInterpolation": "linear",
3845                 "lineWidth": 1,
3846                 "pointSize": 5,
3847                 "scaleDistribution": {
3848                   "type": "linear"
3849                 },
3850                 "showPoints": "never",
3851                 "spanNulls": true,
3852                 "stacking": {
3853                   "group": "A",
3854                   "mode": "none"
3855                 },
3856                 "thresholdsStyle": {
3857                   "mode": "off"
3858                 }
3859               },
3860               "links": [],
3861               "mappings": [],
3862               "min": 0,
3863               "thresholds": {
3864                 "mode": "absolute",
3865                 "steps": [
3866                   {
3867                     "color": "green"
3868                   },
3869                   {
3870                     "color": "red",
3871                     "value": 80
3872                   }
3873                 ]
3874               },
3875               "unit": "ms"
3876             },
3877             "overrides": []
3878           },
3879           "gridPos": {
3880             "h": 9,
3881             "w": 12,
3882             "x": 0,
3883             "y": 30
3884           },
3885           "id": 69,
3886           "options": {
3887             "legend": {
3888               "calcs": [
3889                 "mean",
3890                 "lastNotNull",
3891                 "max"
3892               ],
3893               "displayMode": "list",
3894               "placement": "bottom",
3895               "showLegend": true
3896             },
3897             "tooltip": {
3898               "mode": "single"
3899             }
3900           },
3901           "pluginVersion": "8.1.3",
3902           "targets": [
3903             {
3904               "datasource": {
3905                 "type": "prometheus",
3906                 "uid": "PBFA97CFB590B2093"
3907               },
3908               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3909               "hide": false,
3910               "legendFormat": "{{instance}} - {{quantile}}",
3911               "refId": "A"
3912             }
3913           ],
3914           "title": "Consumer - RequestQueueTimeMs",
3915           "type": "timeseries"
3916         },
3917         {
3918           "datasource": {
3919             "type": "prometheus",
3920             "uid": "PBFA97CFB590B2093"
3921           },
3922           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3923           "fieldConfig": {
3924             "defaults": {
3925               "color": {
3926                 "mode": "palette-classic"
3927               },
3928               "custom": {
3929                 "axisLabel": "",
3930                 "axisPlacement": "auto",
3931                 "barAlignment": 0,
3932                 "drawStyle": "line",
3933                 "fillOpacity": 10,
3934                 "gradientMode": "none",
3935                 "hideFrom": {
3936                   "legend": false,
3937                   "tooltip": false,
3938                   "viz": false
3939                 },
3940                 "lineInterpolation": "linear",
3941                 "lineWidth": 1,
3942                 "pointSize": 5,
3943                 "scaleDistribution": {
3944                   "type": "linear"
3945                 },
3946                 "showPoints": "never",
3947                 "spanNulls": true,
3948                 "stacking": {
3949                   "group": "A",
3950                   "mode": "none"
3951                 },
3952                 "thresholdsStyle": {
3953                   "mode": "off"
3954                 }
3955               },
3956               "links": [],
3957               "mappings": [],
3958               "thresholds": {
3959                 "mode": "absolute",
3960                 "steps": [
3961                   {
3962                     "color": "green"
3963                   },
3964                   {
3965                     "color": "red",
3966                     "value": 80
3967                   }
3968                 ]
3969               },
3970               "unit": "ms"
3971             },
3972             "overrides": []
3973           },
3974           "gridPos": {
3975             "h": 9,
3976             "w": 12,
3977             "x": 12,
3978             "y": 30
3979           },
3980           "id": 70,
3981           "options": {
3982             "legend": {
3983               "calcs": [
3984                 "mean",
3985                 "lastNotNull",
3986                 "max"
3987               ],
3988               "displayMode": "list",
3989               "placement": "bottom",
3990               "showLegend": true
3991             },
3992             "tooltip": {
3993               "mode": "single"
3994             }
3995           },
3996           "pluginVersion": "8.1.3",
3997           "targets": [
3998             {
3999               "datasource": {
4000                 "type": "prometheus",
4001                 "uid": "PBFA97CFB590B2093"
4002               },
4003               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4004               "hide": false,
4005               "legendFormat": "{{instance}} - {{quantile}}",
4006               "refId": "A"
4007             }
4008           ],
4009           "title": "Consumer - LocalTimeMs",
4010           "type": "timeseries"
4011         },
4012         {
4013           "datasource": {
4014             "type": "prometheus",
4015             "uid": "PBFA97CFB590B2093"
4016           },
4017           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4018           "fieldConfig": {
4019             "defaults": {
4020               "color": {
4021                 "mode": "palette-classic"
4022               },
4023               "custom": {
4024                 "axisLabel": "",
4025                 "axisPlacement": "auto",
4026                 "barAlignment": 0,
4027                 "drawStyle": "line",
4028                 "fillOpacity": 10,
4029                 "gradientMode": "none",
4030                 "hideFrom": {
4031                   "legend": false,
4032                   "tooltip": false,
4033                   "viz": false
4034                 },
4035                 "lineInterpolation": "linear",
4036                 "lineWidth": 1,
4037                 "pointSize": 5,
4038                 "scaleDistribution": {
4039                   "type": "linear"
4040                 },
4041                 "showPoints": "never",
4042                 "spanNulls": true,
4043                 "stacking": {
4044                   "group": "A",
4045                   "mode": "none"
4046                 },
4047                 "thresholdsStyle": {
4048                   "mode": "off"
4049                 }
4050               },
4051               "links": [],
4052               "mappings": [],
4053               "thresholds": {
4054                 "mode": "absolute",
4055                 "steps": [
4056                   {
4057                     "color": "green"
4058                   },
4059                   {
4060                     "color": "red",
4061                     "value": 80
4062                   }
4063                 ]
4064               },
4065               "unit": "ms"
4066             },
4067             "overrides": []
4068           },
4069           "gridPos": {
4070             "h": 9,
4071             "w": 8,
4072             "x": 0,
4073             "y": 39
4074           },
4075           "id": 71,
4076           "options": {
4077             "legend": {
4078               "calcs": [
4079                 "mean",
4080                 "lastNotNull",
4081                 "max"
4082               ],
4083               "displayMode": "table",
4084               "placement": "bottom",
4085               "showLegend": true
4086             },
4087             "tooltip": {
4088               "mode": "single"
4089             }
4090           },
4091           "pluginVersion": "8.1.3",
4092           "targets": [
4093             {
4094               "datasource": {
4095                 "type": "prometheus",
4096                 "uid": "PBFA97CFB590B2093"
4097               },
4098               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4099               "hide": false,
4100               "legendFormat": "{{instance}} - {{quantile}}",
4101               "refId": "A"
4102             }
4103           ],
4104           "title": "Consumer - RemoteTimeMs",
4105           "type": "timeseries"
4106         },
4107         {
4108           "datasource": {
4109             "type": "prometheus",
4110             "uid": "PBFA97CFB590B2093"
4111           },
4112           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4113           "fieldConfig": {
4114             "defaults": {
4115               "color": {
4116                 "mode": "palette-classic"
4117               },
4118               "custom": {
4119                 "axisLabel": "",
4120                 "axisPlacement": "auto",
4121                 "barAlignment": 0,
4122                 "drawStyle": "line",
4123                 "fillOpacity": 10,
4124                 "gradientMode": "none",
4125                 "hideFrom": {
4126                   "legend": false,
4127                   "tooltip": false,
4128                   "viz": false
4129                 },
4130                 "lineInterpolation": "linear",
4131                 "lineWidth": 1,
4132                 "pointSize": 5,
4133                 "scaleDistribution": {
4134                   "type": "linear"
4135                 },
4136                 "showPoints": "never",
4137                 "spanNulls": true,
4138                 "stacking": {
4139                   "group": "A",
4140                   "mode": "none"
4141                 },
4142                 "thresholdsStyle": {
4143                   "mode": "off"
4144                 }
4145               },
4146               "links": [],
4147               "mappings": [],
4148               "min": 0,
4149               "thresholds": {
4150                 "mode": "absolute",
4151                 "steps": [
4152                   {
4153                     "color": "green"
4154                   },
4155                   {
4156                     "color": "red",
4157                     "value": 80
4158                   }
4159                 ]
4160               },
4161               "unit": "ms"
4162             },
4163             "overrides": []
4164           },
4165           "gridPos": {
4166             "h": 9,
4167             "w": 8,
4168             "x": 8,
4169             "y": 39
4170           },
4171           "id": 72,
4172           "options": {
4173             "legend": {
4174               "calcs": [
4175                 "mean",
4176                 "lastNotNull",
4177                 "max"
4178               ],
4179               "displayMode": "table",
4180               "placement": "bottom",
4181               "showLegend": true
4182             },
4183             "tooltip": {
4184               "mode": "single"
4185             }
4186           },
4187           "pluginVersion": "8.1.3",
4188           "targets": [
4189             {
4190               "datasource": {
4191                 "type": "prometheus",
4192                 "uid": "PBFA97CFB590B2093"
4193               },
4194               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4195               "hide": false,
4196               "legendFormat": "{{instance}} - {{quantile}}",
4197               "refId": "A"
4198             }
4199           ],
4200           "title": "Consumer - ResponseQueueTimeMs",
4201           "type": "timeseries"
4202         },
4203         {
4204           "datasource": {
4205             "type": "prometheus",
4206             "uid": "PBFA97CFB590B2093"
4207           },
4208           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4209           "fieldConfig": {
4210             "defaults": {
4211               "color": {
4212                 "mode": "palette-classic"
4213               },
4214               "custom": {
4215                 "axisLabel": "",
4216                 "axisPlacement": "auto",
4217                 "barAlignment": 0,
4218                 "drawStyle": "line",
4219                 "fillOpacity": 10,
4220                 "gradientMode": "none",
4221                 "hideFrom": {
4222                   "legend": false,
4223                   "tooltip": false,
4224                   "viz": false
4225                 },
4226                 "lineInterpolation": "linear",
4227                 "lineWidth": 1,
4228                 "pointSize": 5,
4229                 "scaleDistribution": {
4230                   "type": "linear"
4231                 },
4232                 "showPoints": "never",
4233                 "spanNulls": true,
4234                 "stacking": {
4235                   "group": "A",
4236                   "mode": "none"
4237                 },
4238                 "thresholdsStyle": {
4239                   "mode": "off"
4240                 }
4241               },
4242               "links": [],
4243               "mappings": [],
4244               "min": 0,
4245               "thresholds": {
4246                 "mode": "absolute",
4247                 "steps": [
4248                   {
4249                     "color": "green"
4250                   },
4251                   {
4252                     "color": "red",
4253                     "value": 80
4254                   }
4255                 ]
4256               },
4257               "unit": "ms"
4258             },
4259             "overrides": []
4260           },
4261           "gridPos": {
4262             "h": 9,
4263             "w": 8,
4264             "x": 16,
4265             "y": 39
4266           },
4267           "id": 73,
4268           "options": {
4269             "legend": {
4270               "calcs": [
4271                 "mean",
4272                 "lastNotNull",
4273                 "max"
4274               ],
4275               "displayMode": "table",
4276               "placement": "bottom",
4277               "showLegend": true
4278             },
4279             "tooltip": {
4280               "mode": "single"
4281             }
4282           },
4283           "pluginVersion": "8.1.3",
4284           "targets": [
4285             {
4286               "datasource": {
4287                 "type": "prometheus",
4288                 "uid": "PBFA97CFB590B2093"
4289               },
4290               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4291               "hide": false,
4292               "legendFormat": "{{instance}} - {{quantile}}",
4293               "refId": "A"
4294             }
4295           ],
4296           "title": "Consumer - ResponseSendTimeMs",
4297           "type": "timeseries"
4298         }
4299       ],
4300       "targets": [
4301         {
4302           "datasource": {
4303             "type": "datasource",
4304             "uid": "grafana"
4305           },
4306           "refId": "A"
4307         }
4308       ],
4309       "title": "Consumer Performance",
4310       "type": "row"
4311     },
4312     {
4313       "collapsed": true,
4314       "datasource": {
4315         "type": "datasource",
4316         "uid": "grafana"
4317       },
4318       "gridPos": {
4319         "h": 1,
4320         "w": 24,
4321         "x": 0,
4322         "y": 70
4323       },
4324       "id": 66,
4325       "panels": [
4326         {
4327           "datasource": {
4328             "type": "prometheus",
4329             "uid": "PBFA97CFB590B2093"
4330           },
4331           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4332           "fieldConfig": {
4333             "defaults": {
4334               "color": {
4335                 "mode": "palette-classic"
4336               },
4337               "custom": {
4338                 "axisLabel": "",
4339                 "axisPlacement": "auto",
4340                 "barAlignment": 0,
4341                 "drawStyle": "line",
4342                 "fillOpacity": 10,
4343                 "gradientMode": "none",
4344                 "hideFrom": {
4345                   "legend": false,
4346                   "tooltip": false,
4347                   "viz": false
4348                 },
4349                 "lineInterpolation": "linear",
4350                 "lineWidth": 1,
4351                 "pointSize": 5,
4352                 "scaleDistribution": {
4353                   "type": "linear"
4354                 },
4355                 "showPoints": "never",
4356                 "spanNulls": true,
4357                 "stacking": {
4358                   "group": "A",
4359                   "mode": "none"
4360                 },
4361                 "thresholdsStyle": {
4362                   "mode": "off"
4363                 }
4364               },
4365               "links": [],
4366               "mappings": [],
4367               "min": 0,
4368               "thresholds": {
4369                 "mode": "absolute",
4370                 "steps": [
4371                   {
4372                     "color": "green"
4373                   },
4374                   {
4375                     "color": "red",
4376                     "value": 80
4377                   }
4378                 ]
4379               },
4380               "unit": "ms"
4381             },
4382             "overrides": []
4383           },
4384           "gridPos": {
4385             "h": 9,
4386             "w": 12,
4387             "x": 0,
4388             "y": 31
4389           },
4390           "id": 74,
4391           "options": {
4392             "legend": {
4393               "calcs": [
4394                 "mean",
4395                 "lastNotNull",
4396                 "max"
4397               ],
4398               "displayMode": "list",
4399               "placement": "bottom",
4400               "showLegend": true
4401             },
4402             "tooltip": {
4403               "mode": "single"
4404             }
4405           },
4406           "pluginVersion": "8.1.3",
4407           "targets": [
4408             {
4409               "datasource": {
4410                 "type": "prometheus",
4411                 "uid": "PBFA97CFB590B2093"
4412               },
4413               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4414               "hide": false,
4415               "legendFormat": "{{instance}} - {{quantile}}",
4416               "refId": "A"
4417             }
4418           ],
4419           "title": "FetchFollower - RequestQueueTimeMs",
4420           "type": "timeseries"
4421         },
4422         {
4423           "datasource": {
4424             "type": "prometheus",
4425             "uid": "PBFA97CFB590B2093"
4426           },
4427           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4428           "fieldConfig": {
4429             "defaults": {
4430               "color": {
4431                 "mode": "palette-classic"
4432               },
4433               "custom": {
4434                 "axisLabel": "",
4435                 "axisPlacement": "auto",
4436                 "barAlignment": 0,
4437                 "drawStyle": "line",
4438                 "fillOpacity": 10,
4439                 "gradientMode": "none",
4440                 "hideFrom": {
4441                   "legend": false,
4442                   "tooltip": false,
4443                   "viz": false
4444                 },
4445                 "lineInterpolation": "linear",
4446                 "lineWidth": 1,
4447                 "pointSize": 5,
4448                 "scaleDistribution": {
4449                   "type": "linear"
4450                 },
4451                 "showPoints": "never",
4452                 "spanNulls": true,
4453                 "stacking": {
4454                   "group": "A",
4455                   "mode": "none"
4456                 },
4457                 "thresholdsStyle": {
4458                   "mode": "off"
4459                 }
4460               },
4461               "links": [],
4462               "mappings": [],
4463               "min": 0,
4464               "thresholds": {
4465                 "mode": "absolute",
4466                 "steps": [
4467                   {
4468                     "color": "green"
4469                   },
4470                   {
4471                     "color": "red",
4472                     "value": 80
4473                   }
4474                 ]
4475               },
4476               "unit": "ms"
4477             },
4478             "overrides": []
4479           },
4480           "gridPos": {
4481             "h": 9,
4482             "w": 12,
4483             "x": 12,
4484             "y": 31
4485           },
4486           "id": 75,
4487           "options": {
4488             "legend": {
4489               "calcs": [
4490                 "mean",
4491                 "lastNotNull",
4492                 "max"
4493               ],
4494               "displayMode": "list",
4495               "placement": "bottom",
4496               "showLegend": true
4497             },
4498             "tooltip": {
4499               "mode": "single"
4500             }
4501           },
4502           "pluginVersion": "8.1.3",
4503           "targets": [
4504             {
4505               "datasource": {
4506                 "type": "prometheus",
4507                 "uid": "PBFA97CFB590B2093"
4508               },
4509               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4510               "hide": false,
4511               "legendFormat": "{{instance}} - {{quantile}}",
4512               "refId": "A"
4513             }
4514           ],
4515           "title": "FetchFollower - LocalTimeMs",
4516           "type": "timeseries"
4517         },
4518         {
4519           "datasource": {
4520             "type": "prometheus",
4521             "uid": "PBFA97CFB590B2093"
4522           },
4523           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4524           "fieldConfig": {
4525             "defaults": {
4526               "color": {
4527                 "mode": "palette-classic"
4528               },
4529               "custom": {
4530                 "axisLabel": "",
4531                 "axisPlacement": "auto",
4532                 "barAlignment": 0,
4533                 "drawStyle": "line",
4534                 "fillOpacity": 10,
4535                 "gradientMode": "none",
4536                 "hideFrom": {
4537                   "legend": false,
4538                   "tooltip": false,
4539                   "viz": false
4540                 },
4541                 "lineInterpolation": "linear",
4542                 "lineWidth": 1,
4543                 "pointSize": 5,
4544                 "scaleDistribution": {
4545                   "type": "linear"
4546                 },
4547                 "showPoints": "never",
4548                 "spanNulls": true,
4549                 "stacking": {
4550                   "group": "A",
4551                   "mode": "none"
4552                 },
4553                 "thresholdsStyle": {
4554                   "mode": "off"
4555                 }
4556               },
4557               "links": [],
4558               "mappings": [],
4559               "thresholds": {
4560                 "mode": "absolute",
4561                 "steps": [
4562                   {
4563                     "color": "green"
4564                   },
4565                   {
4566                     "color": "red",
4567                     "value": 80
4568                   }
4569                 ]
4570               },
4571               "unit": "ms"
4572             },
4573             "overrides": []
4574           },
4575           "gridPos": {
4576             "h": 9,
4577             "w": 8,
4578             "x": 0,
4579             "y": 40
4580           },
4581           "id": 76,
4582           "options": {
4583             "legend": {
4584               "calcs": [
4585                 "mean",
4586                 "lastNotNull",
4587                 "max"
4588               ],
4589               "displayMode": "table",
4590               "placement": "bottom",
4591               "showLegend": true
4592             },
4593             "tooltip": {
4594               "mode": "single"
4595             }
4596           },
4597           "pluginVersion": "8.1.3",
4598           "targets": [
4599             {
4600               "datasource": {
4601                 "type": "prometheus",
4602                 "uid": "PBFA97CFB590B2093"
4603               },
4604               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4605               "hide": false,
4606               "legendFormat": "{{instance}} - {{quantile}}",
4607               "refId": "A"
4608             }
4609           ],
4610           "title": "FetchFollower - RemoteTimeMs",
4611           "type": "timeseries"
4612         },
4613         {
4614           "datasource": {
4615             "type": "prometheus",
4616             "uid": "PBFA97CFB590B2093"
4617           },
4618           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4619           "fieldConfig": {
4620             "defaults": {
4621               "color": {
4622                 "mode": "palette-classic"
4623               },
4624               "custom": {
4625                 "axisLabel": "",
4626                 "axisPlacement": "auto",
4627                 "barAlignment": 0,
4628                 "drawStyle": "line",
4629                 "fillOpacity": 10,
4630                 "gradientMode": "none",
4631                 "hideFrom": {
4632                   "legend": false,
4633                   "tooltip": false,
4634                   "viz": false
4635                 },
4636                 "lineInterpolation": "linear",
4637                 "lineWidth": 1,
4638                 "pointSize": 5,
4639                 "scaleDistribution": {
4640                   "type": "linear"
4641                 },
4642                 "showPoints": "never",
4643                 "spanNulls": true,
4644                 "stacking": {
4645                   "group": "A",
4646                   "mode": "none"
4647                 },
4648                 "thresholdsStyle": {
4649                   "mode": "off"
4650                 }
4651               },
4652               "links": [],
4653               "mappings": [],
4654               "thresholds": {
4655                 "mode": "absolute",
4656                 "steps": [
4657                   {
4658                     "color": "green"
4659                   },
4660                   {
4661                     "color": "red",
4662                     "value": 80
4663                   }
4664                 ]
4665               },
4666               "unit": "ms"
4667             },
4668             "overrides": []
4669           },
4670           "gridPos": {
4671             "h": 9,
4672             "w": 8,
4673             "x": 8,
4674             "y": 40
4675           },
4676           "id": 77,
4677           "options": {
4678             "legend": {
4679               "calcs": [
4680                 "mean",
4681                 "lastNotNull",
4682                 "max"
4683               ],
4684               "displayMode": "table",
4685               "placement": "bottom",
4686               "showLegend": true
4687             },
4688             "tooltip": {
4689               "mode": "single"
4690             }
4691           },
4692           "pluginVersion": "8.1.3",
4693           "targets": [
4694             {
4695               "datasource": {
4696                 "type": "prometheus",
4697                 "uid": "PBFA97CFB590B2093"
4698               },
4699               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4700               "hide": false,
4701               "legendFormat": "{{instance}} - {{quantile}}",
4702               "refId": "A"
4703             }
4704           ],
4705           "title": "FetchFollower - ResponseQueueTimeMs",
4706           "type": "timeseries"
4707         },
4708         {
4709           "datasource": {
4710             "type": "prometheus",
4711             "uid": "PBFA97CFB590B2093"
4712           },
4713           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4714           "fieldConfig": {
4715             "defaults": {
4716               "color": {
4717                 "mode": "palette-classic"
4718               },
4719               "custom": {
4720                 "axisLabel": "",
4721                 "axisPlacement": "auto",
4722                 "barAlignment": 0,
4723                 "drawStyle": "line",
4724                 "fillOpacity": 10,
4725                 "gradientMode": "none",
4726                 "hideFrom": {
4727                   "legend": false,
4728                   "tooltip": false,
4729                   "viz": false
4730                 },
4731                 "lineInterpolation": "linear",
4732                 "lineWidth": 1,
4733                 "pointSize": 5,
4734                 "scaleDistribution": {
4735                   "type": "linear"
4736                 },
4737                 "showPoints": "never",
4738                 "spanNulls": true,
4739                 "stacking": {
4740                   "group": "A",
4741                   "mode": "none"
4742                 },
4743                 "thresholdsStyle": {
4744                   "mode": "off"
4745                 }
4746               },
4747               "links": [],
4748               "mappings": [],
4749               "min": 0,
4750               "thresholds": {
4751                 "mode": "absolute",
4752                 "steps": [
4753                   {
4754                     "color": "green"
4755                   },
4756                   {
4757                     "color": "red",
4758                     "value": 80
4759                   }
4760                 ]
4761               },
4762               "unit": "ms"
4763             },
4764             "overrides": []
4765           },
4766           "gridPos": {
4767             "h": 9,
4768             "w": 8,
4769             "x": 16,
4770             "y": 40
4771           },
4772           "id": 78,
4773           "options": {
4774             "legend": {
4775               "calcs": [
4776                 "mean",
4777                 "lastNotNull",
4778                 "max"
4779               ],
4780               "displayMode": "table",
4781               "placement": "bottom",
4782               "showLegend": true
4783             },
4784             "tooltip": {
4785               "mode": "single"
4786             }
4787           },
4788           "pluginVersion": "8.1.3",
4789           "targets": [
4790             {
4791               "datasource": {
4792                 "type": "prometheus",
4793                 "uid": "PBFA97CFB590B2093"
4794               },
4795               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4796               "hide": false,
4797               "legendFormat": "{{instance}} - {{quantile}}",
4798               "refId": "A"
4799             }
4800           ],
4801           "title": "FetchFollower - ResponseSendTimeMs",
4802           "type": "timeseries"
4803         }
4804       ],
4805       "targets": [
4806         {
4807           "datasource": {
4808             "type": "datasource",
4809             "uid": "grafana"
4810           },
4811           "refId": "A"
4812         }
4813       ],
4814       "title": "Fetch Follower Performance",
4815       "type": "row"
4816     },
4817     {
4818       "collapsed": true,
4819       "datasource": {
4820         "type": "datasource",
4821         "uid": "grafana"
4822       },
4823       "gridPos": {
4824         "h": 1,
4825         "w": 24,
4826         "x": 0,
4827         "y": 71
4828       },
4829       "id": 102,
4830       "panels": [
4831         {
4832           "datasource": {
4833             "type": "prometheus",
4834             "uid": "PBFA97CFB590B2093"
4835           },
4836           "fieldConfig": {
4837             "defaults": {
4838               "color": {
4839                 "mode": "palette-classic"
4840               },
4841               "custom": {
4842                 "axisLabel": "",
4843                 "axisPlacement": "auto",
4844                 "barAlignment": 0,
4845                 "drawStyle": "line",
4846                 "fillOpacity": 10,
4847                 "gradientMode": "none",
4848                 "hideFrom": {
4849                   "legend": false,
4850                   "tooltip": false,
4851                   "viz": false
4852                 },
4853                 "lineInterpolation": "linear",
4854                 "lineWidth": 1,
4855                 "pointSize": 5,
4856                 "scaleDistribution": {
4857                   "type": "linear"
4858                 },
4859                 "showPoints": "never",
4860                 "spanNulls": true,
4861                 "stacking": {
4862                   "group": "A",
4863                   "mode": "none"
4864                 },
4865                 "thresholdsStyle": {
4866                   "mode": "off"
4867                 }
4868               },
4869               "links": [],
4870               "mappings": [],
4871               "thresholds": {
4872                 "mode": "absolute",
4873                 "steps": [
4874                   {
4875                     "color": "green"
4876                   },
4877                   {
4878                     "color": "red",
4879                     "value": 80
4880                   }
4881                 ]
4882               },
4883               "unit": "short"
4884             },
4885             "overrides": []
4886           },
4887           "gridPos": {
4888             "h": 8,
4889             "w": 12,
4890             "x": 0,
4891             "y": 32
4892           },
4893           "id": 98,
4894           "options": {
4895             "legend": {
4896               "calcs": [
4897                 "mean",
4898                 "lastNotNull",
4899                 "max"
4900               ],
4901               "displayMode": "table",
4902               "placement": "bottom",
4903               "showLegend": true
4904             },
4905             "tooltip": {
4906               "mode": "single"
4907             }
4908           },
4909           "pluginVersion": "8.1.3",
4910           "targets": [
4911             {
4912               "datasource": {
4913                 "type": "prometheus",
4914                 "uid": "PBFA97CFB590B2093"
4915               },
4916               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4917               "interval": "",
4918               "legendFormat": "{{listener}}",
4919               "refId": "A"
4920             }
4921           ],
4922           "title": "Connections count per listener",
4923           "type": "timeseries"
4924         },
4925         {
4926           "datasource": {
4927             "type": "prometheus",
4928             "uid": "PBFA97CFB590B2093"
4929           },
4930           "fieldConfig": {
4931             "defaults": {
4932               "color": {
4933                 "mode": "palette-classic"
4934               },
4935               "custom": {
4936                 "axisLabel": "",
4937                 "axisPlacement": "auto",
4938                 "barAlignment": 0,
4939                 "drawStyle": "line",
4940                 "fillOpacity": 10,
4941                 "gradientMode": "none",
4942                 "hideFrom": {
4943                   "legend": false,
4944                   "tooltip": false,
4945                   "viz": false
4946                 },
4947                 "lineInterpolation": "linear",
4948                 "lineWidth": 1,
4949                 "pointSize": 5,
4950                 "scaleDistribution": {
4951                   "type": "linear"
4952                 },
4953                 "showPoints": "never",
4954                 "spanNulls": true,
4955                 "stacking": {
4956                   "group": "A",
4957                   "mode": "none"
4958                 },
4959                 "thresholdsStyle": {
4960                   "mode": "off"
4961                 }
4962               },
4963               "links": [],
4964               "mappings": [],
4965               "thresholds": {
4966                 "mode": "absolute",
4967                 "steps": [
4968                   {
4969                     "color": "green"
4970                   },
4971                   {
4972                     "color": "red",
4973                     "value": 80
4974                   }
4975                 ]
4976               },
4977               "unit": "short"
4978             },
4979             "overrides": []
4980           },
4981           "gridPos": {
4982             "h": 8,
4983             "w": 12,
4984             "x": 12,
4985             "y": 32
4986           },
4987           "id": 100,
4988           "options": {
4989             "legend": {
4990               "calcs": [
4991                 "mean",
4992                 "lastNotNull",
4993                 "max"
4994               ],
4995               "displayMode": "table",
4996               "placement": "bottom",
4997               "showLegend": true
4998             },
4999             "tooltip": {
5000               "mode": "single"
5001             }
5002           },
5003           "pluginVersion": "8.1.3",
5004           "targets": [
5005             {
5006               "datasource": {
5007                 "type": "prometheus",
5008                 "uid": "PBFA97CFB590B2093"
5009               },
5010               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5011               "interval": "",
5012               "legendFormat": "{{instance}}",
5013               "refId": "A"
5014             }
5015           ],
5016           "title": "Connections count per broker",
5017           "type": "timeseries"
5018         },
5019         {
5020           "datasource": {
5021             "type": "prometheus",
5022             "uid": "PBFA97CFB590B2093"
5023           },
5024           "fieldConfig": {
5025             "defaults": {
5026               "color": {
5027                 "mode": "palette-classic"
5028               },
5029               "custom": {
5030                 "axisLabel": "",
5031                 "axisPlacement": "auto",
5032                 "barAlignment": 0,
5033                 "drawStyle": "line",
5034                 "fillOpacity": 10,
5035                 "gradientMode": "none",
5036                 "hideFrom": {
5037                   "legend": false,
5038                   "tooltip": false,
5039                   "viz": false
5040                 },
5041                 "lineInterpolation": "linear",
5042                 "lineWidth": 1,
5043                 "pointSize": 5,
5044                 "scaleDistribution": {
5045                   "type": "linear"
5046                 },
5047                 "showPoints": "never",
5048                 "spanNulls": true,
5049                 "stacking": {
5050                   "group": "A",
5051                   "mode": "none"
5052                 },
5053                 "thresholdsStyle": {
5054                   "mode": "off"
5055                 }
5056               },
5057               "links": [],
5058               "mappings": [],
5059               "thresholds": {
5060                 "mode": "absolute",
5061                 "steps": [
5062                   {
5063                     "color": "green"
5064                   },
5065                   {
5066                     "color": "red",
5067                     "value": 80
5068                   }
5069                 ]
5070               },
5071               "unit": "short"
5072             },
5073             "overrides": []
5074           },
5075           "gridPos": {
5076             "h": 8,
5077             "w": 12,
5078             "x": 0,
5079             "y": 40
5080           },
5081           "id": 104,
5082           "options": {
5083             "legend": {
5084               "calcs": [
5085                 "mean",
5086                 "lastNotNull",
5087                 "max"
5088               ],
5089               "displayMode": "table",
5090               "placement": "bottom",
5091               "showLegend": true
5092             },
5093             "tooltip": {
5094               "mode": "single"
5095             }
5096           },
5097           "pluginVersion": "8.1.3",
5098           "targets": [
5099             {
5100               "datasource": {
5101                 "type": "prometheus",
5102                 "uid": "PBFA97CFB590B2093"
5103               },
5104               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5105               "interval": "",
5106               "legendFormat": "{{listener}}",
5107               "refId": "A"
5108             }
5109           ],
5110           "title": "Connections creation rate per listener",
5111           "type": "timeseries"
5112         },
5113         {
5114           "datasource": {
5115             "type": "prometheus",
5116             "uid": "PBFA97CFB590B2093"
5117           },
5118           "fieldConfig": {
5119             "defaults": {
5120               "color": {
5121                 "mode": "palette-classic"
5122               },
5123               "custom": {
5124                 "axisLabel": "",
5125                 "axisPlacement": "auto",
5126                 "barAlignment": 0,
5127                 "drawStyle": "line",
5128                 "fillOpacity": 10,
5129                 "gradientMode": "none",
5130                 "hideFrom": {
5131                   "legend": false,
5132                   "tooltip": false,
5133                   "viz": false
5134                 },
5135                 "lineInterpolation": "linear",
5136                 "lineWidth": 1,
5137                 "pointSize": 5,
5138                 "scaleDistribution": {
5139                   "type": "linear"
5140                 },
5141                 "showPoints": "never",
5142                 "spanNulls": true,
5143                 "stacking": {
5144                   "group": "A",
5145                   "mode": "none"
5146                 },
5147                 "thresholdsStyle": {
5148                   "mode": "off"
5149                 }
5150               },
5151               "links": [],
5152               "mappings": [],
5153               "thresholds": {
5154                 "mode": "absolute",
5155                 "steps": [
5156                   {
5157                     "color": "green"
5158                   },
5159                   {
5160                     "color": "red",
5161                     "value": 80
5162                   }
5163                 ]
5164               },
5165               "unit": "short"
5166             },
5167             "overrides": []
5168           },
5169           "gridPos": {
5170             "h": 8,
5171             "w": 12,
5172             "x": 12,
5173             "y": 40
5174           },
5175           "id": 106,
5176           "options": {
5177             "legend": {
5178               "calcs": [
5179                 "mean",
5180                 "lastNotNull",
5181                 "max"
5182               ],
5183               "displayMode": "table",
5184               "placement": "bottom",
5185               "showLegend": true
5186             },
5187             "tooltip": {
5188               "mode": "single"
5189             }
5190           },
5191           "pluginVersion": "8.1.3",
5192           "targets": [
5193             {
5194               "datasource": {
5195                 "type": "prometheus",
5196                 "uid": "PBFA97CFB590B2093"
5197               },
5198               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5199               "interval": "",
5200               "legendFormat": "{{instance}}",
5201               "refId": "A"
5202             }
5203           ],
5204           "title": "Connections creation rate per instance",
5205           "type": "timeseries"
5206         },
5207         {
5208           "datasource": {
5209             "type": "prometheus",
5210             "uid": "PBFA97CFB590B2093"
5211           },
5212           "fieldConfig": {
5213             "defaults": {
5214               "color": {
5215                 "mode": "palette-classic"
5216               },
5217               "custom": {
5218                 "axisLabel": "",
5219                 "axisPlacement": "auto",
5220                 "barAlignment": 0,
5221                 "drawStyle": "line",
5222                 "fillOpacity": 10,
5223                 "gradientMode": "none",
5224                 "hideFrom": {
5225                   "legend": false,
5226                   "tooltip": false,
5227                   "viz": false
5228                 },
5229                 "lineInterpolation": "linear",
5230                 "lineWidth": 1,
5231                 "pointSize": 5,
5232                 "scaleDistribution": {
5233                   "type": "linear"
5234                 },
5235                 "showPoints": "never",
5236                 "spanNulls": true,
5237                 "stacking": {
5238                   "group": "A",
5239                   "mode": "none"
5240                 },
5241                 "thresholdsStyle": {
5242                   "mode": "off"
5243                 }
5244               },
5245               "links": [],
5246               "mappings": [],
5247               "thresholds": {
5248                 "mode": "absolute",
5249                 "steps": [
5250                   {
5251                     "color": "green"
5252                   },
5253                   {
5254                     "color": "red",
5255                     "value": 80
5256                   }
5257                 ]
5258               },
5259               "unit": "short"
5260             },
5261             "overrides": []
5262           },
5263           "gridPos": {
5264             "h": 8,
5265             "w": 12,
5266             "x": 0,
5267             "y": 48
5268           },
5269           "id": 108,
5270           "options": {
5271             "legend": {
5272               "calcs": [
5273                 "mean",
5274                 "lastNotNull",
5275                 "max"
5276               ],
5277               "displayMode": "table",
5278               "placement": "bottom",
5279               "showLegend": true
5280             },
5281             "tooltip": {
5282               "mode": "single"
5283             }
5284           },
5285           "pluginVersion": "8.1.3",
5286           "targets": [
5287             {
5288               "datasource": {
5289                 "type": "prometheus",
5290                 "uid": "PBFA97CFB590B2093"
5291               },
5292               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5293               "interval": "",
5294               "legendFormat": "{{listener}}",
5295               "refId": "A"
5296             }
5297           ],
5298           "title": "Connections close rate per listener",
5299           "type": "timeseries"
5300         },
5301         {
5302           "datasource": {
5303             "type": "prometheus",
5304             "uid": "PBFA97CFB590B2093"
5305           },
5306           "fieldConfig": {
5307             "defaults": {
5308               "color": {
5309                 "mode": "palette-classic"
5310               },
5311               "custom": {
5312                 "axisLabel": "",
5313                 "axisPlacement": "auto",
5314                 "barAlignment": 0,
5315                 "drawStyle": "line",
5316                 "fillOpacity": 10,
5317                 "gradientMode": "none",
5318                 "hideFrom": {
5319                   "legend": false,
5320                   "tooltip": false,
5321                   "viz": false
5322                 },
5323                 "lineInterpolation": "linear",
5324                 "lineWidth": 1,
5325                 "pointSize": 5,
5326                 "scaleDistribution": {
5327                   "type": "linear"
5328                 },
5329                 "showPoints": "never",
5330                 "spanNulls": true,
5331                 "stacking": {
5332                   "group": "A",
5333                   "mode": "none"
5334                 },
5335                 "thresholdsStyle": {
5336                   "mode": "off"
5337                 }
5338               },
5339               "links": [],
5340               "mappings": [],
5341               "thresholds": {
5342                 "mode": "absolute",
5343                 "steps": [
5344                   {
5345                     "color": "green"
5346                   },
5347                   {
5348                     "color": "red",
5349                     "value": 80
5350                   }
5351                 ]
5352               },
5353               "unit": "short"
5354             },
5355             "overrides": []
5356           },
5357           "gridPos": {
5358             "h": 8,
5359             "w": 12,
5360             "x": 12,
5361             "y": 48
5362           },
5363           "id": 110,
5364           "options": {
5365             "legend": {
5366               "calcs": [
5367                 "mean",
5368                 "lastNotNull",
5369                 "max"
5370               ],
5371               "displayMode": "table",
5372               "placement": "bottom",
5373               "showLegend": true
5374             },
5375             "tooltip": {
5376               "mode": "single"
5377             }
5378           },
5379           "pluginVersion": "8.1.3",
5380           "targets": [
5381             {
5382               "datasource": {
5383                 "type": "prometheus",
5384                 "uid": "PBFA97CFB590B2093"
5385               },
5386               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5387               "interval": "",
5388               "legendFormat": "{{instance}}",
5389               "refId": "A"
5390             }
5391           ],
5392           "title": "Connections close rate per instance",
5393           "type": "timeseries"
5394         },
5395         {
5396           "datasource": {
5397             "type": "prometheus",
5398             "uid": "PBFA97CFB590B2093"
5399           },
5400           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5401           "fieldConfig": {
5402             "defaults": {
5403               "color": {
5404                 "mode": "palette-classic"
5405               },
5406               "custom": {
5407                 "axisLabel": "",
5408                 "axisPlacement": "auto",
5409                 "barAlignment": 0,
5410                 "drawStyle": "line",
5411                 "fillOpacity": 10,
5412                 "gradientMode": "none",
5413                 "hideFrom": {
5414                   "legend": false,
5415                   "tooltip": false,
5416                   "viz": false
5417                 },
5418                 "lineInterpolation": "linear",
5419                 "lineWidth": 1,
5420                 "pointSize": 5,
5421                 "scaleDistribution": {
5422                   "type": "linear"
5423                 },
5424                 "showPoints": "never",
5425                 "spanNulls": true,
5426                 "stacking": {
5427                   "group": "A",
5428                   "mode": "none"
5429                 },
5430                 "thresholdsStyle": {
5431                   "mode": "off"
5432                 }
5433               },
5434               "links": [],
5435               "mappings": [],
5436               "min": 0,
5437               "thresholds": {
5438                 "mode": "absolute",
5439                 "steps": [
5440                   {
5441                     "color": "green"
5442                   },
5443                   {
5444                     "color": "red",
5445                     "value": 80
5446                   }
5447                 ]
5448               },
5449               "unit": "percent"
5450             },
5451             "overrides": []
5452           },
5453           "gridPos": {
5454             "h": 8,
5455             "w": 12,
5456             "x": 0,
5457             "y": 56
5458           },
5459           "id": 112,
5460           "options": {
5461             "legend": {
5462               "calcs": [
5463                 "mean",
5464                 "lastNotNull",
5465                 "max"
5466               ],
5467               "displayMode": "table",
5468               "placement": "bottom",
5469               "showLegend": true
5470             },
5471             "tooltip": {
5472               "mode": "single"
5473             }
5474           },
5475           "pluginVersion": "8.1.3",
5476           "targets": [
5477             {
5478               "datasource": {
5479                 "type": "prometheus",
5480                 "uid": "PBFA97CFB590B2093"
5481               },
5482               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5483               "interval": "",
5484               "legendFormat": "{{instance}} - {{listener}}",
5485               "refId": "A"
5486             }
5487           ],
5488           "title": "Acceptor Blocked Percentage",
5489           "type": "timeseries"
5490         },
5491         {
5492           "datasource": {
5493             "type": "prometheus",
5494             "uid": "PBFA97CFB590B2093"
5495           },
5496           "fieldConfig": {
5497             "defaults": {
5498               "color": {
5499                 "mode": "palette-classic"
5500               },
5501               "custom": {
5502                 "axisLabel": "",
5503                 "axisPlacement": "auto",
5504                 "barAlignment": 0,
5505                 "drawStyle": "line",
5506                 "fillOpacity": 10,
5507                 "gradientMode": "none",
5508                 "hideFrom": {
5509                   "legend": false,
5510                   "tooltip": false,
5511                   "viz": false
5512                 },
5513                 "lineInterpolation": "linear",
5514                 "lineWidth": 1,
5515                 "pointSize": 5,
5516                 "scaleDistribution": {
5517                   "type": "linear"
5518                 },
5519                 "showPoints": "never",
5520                 "spanNulls": true,
5521                 "stacking": {
5522                   "group": "A",
5523                   "mode": "none"
5524                 },
5525                 "thresholdsStyle": {
5526                   "mode": "off"
5527                 }
5528               },
5529               "links": [],
5530               "mappings": [],
5531               "thresholds": {
5532                 "mode": "absolute",
5533                 "steps": [
5534                   {
5535                     "color": "green"
5536                   },
5537                   {
5538                     "color": "red",
5539                     "value": 80
5540                   }
5541                 ]
5542               },
5543               "unit": "short"
5544             },
5545             "overrides": []
5546           },
5547           "gridPos": {
5548             "h": 8,
5549             "w": 12,
5550             "x": 12,
5551             "y": 56
5552           },
5553           "id": 114,
5554           "options": {
5555             "legend": {
5556               "calcs": [
5557                 "mean",
5558                 "lastNotNull",
5559                 "max"
5560               ],
5561               "displayMode": "table",
5562               "placement": "bottom",
5563               "showLegend": true
5564             },
5565             "tooltip": {
5566               "mode": "single"
5567             }
5568           },
5569           "pluginVersion": "8.1.3",
5570           "targets": [
5571             {
5572               "datasource": {
5573                 "type": "prometheus",
5574                 "uid": "PBFA97CFB590B2093"
5575               },
5576               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5577               "interval": "",
5578               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5579               "refId": "A"
5580             }
5581           ],
5582           "title": "Connections per client version",
5583           "type": "timeseries"
5584         }
5585       ],
5586       "targets": [
5587         {
5588           "datasource": {
5589             "type": "datasource",
5590             "uid": "grafana"
5591           },
5592           "refId": "A"
5593         }
5594       ],
5595       "title": "Connections",
5596       "type": "row"
5597     },
5598     {
5599       "collapsed": true,
5600       "datasource": {
5601         "type": "datasource",
5602         "uid": "grafana"
5603       },
5604       "gridPos": {
5605         "h": 1,
5606         "w": 24,
5607         "x": 0,
5608         "y": 72
5609       },
5610       "id": 120,
5611       "panels": [
5612         {
5613           "datasource": {
5614             "type": "prometheus",
5615             "uid": "PBFA97CFB590B2093"
5616           },
5617           "description": "Number of consumer groups per group coordinator",
5618           "fieldConfig": {
5619             "defaults": {
5620               "color": {
5621                 "mode": "palette-classic"
5622               },
5623               "custom": {
5624                 "axisLabel": "",
5625                 "axisPlacement": "auto",
5626                 "barAlignment": 0,
5627                 "drawStyle": "line",
5628                 "fillOpacity": 10,
5629                 "gradientMode": "none",
5630                 "hideFrom": {
5631                   "legend": false,
5632                   "tooltip": false,
5633                   "viz": false
5634                 },
5635                 "lineInterpolation": "linear",
5636                 "lineWidth": 1,
5637                 "pointSize": 5,
5638                 "scaleDistribution": {
5639                   "type": "linear"
5640                 },
5641                 "showPoints": "never",
5642                 "spanNulls": true,
5643                 "stacking": {
5644                   "group": "A",
5645                   "mode": "none"
5646                 },
5647                 "thresholdsStyle": {
5648                   "mode": "off"
5649                 }
5650               },
5651               "links": [],
5652               "mappings": [],
5653               "thresholds": {
5654                 "mode": "absolute",
5655                 "steps": [
5656                   {
5657                     "color": "green"
5658                   },
5659                   {
5660                     "color": "red",
5661                     "value": 80
5662                   }
5663                 ]
5664               },
5665               "unit": "short"
5666             },
5667             "overrides": []
5668           },
5669           "gridPos": {
5670             "h": 8,
5671             "w": 12,
5672             "x": 0,
5673             "y": 33
5674           },
5675           "id": 116,
5676           "options": {
5677             "legend": {
5678               "calcs": [
5679                 "mean",
5680                 "lastNotNull",
5681                 "max"
5682               ],
5683               "displayMode": "table",
5684               "placement": "bottom",
5685               "showLegend": true
5686             },
5687             "tooltip": {
5688               "mode": "single"
5689             }
5690           },
5691           "pluginVersion": "8.1.3",
5692           "targets": [
5693             {
5694               "datasource": {
5695                 "type": "prometheus",
5696                 "uid": "PBFA97CFB590B2093"
5697               },
5698               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5699               "instant": false,
5700               "interval": "",
5701               "legendFormat": "{{instance}}",
5702               "refId": "A"
5703             }
5704           ],
5705           "title": "Consumer groups number per coordinator",
5706           "type": "timeseries"
5707         },
5708         {
5709           "datasource": {
5710             "type": "prometheus",
5711             "uid": "PBFA97CFB590B2093"
5712           },
5713           "description": "Number of consumer group per state",
5714           "fieldConfig": {
5715             "defaults": {
5716               "color": {
5717                 "mode": "palette-classic"
5718               },
5719               "custom": {
5720                 "axisLabel": "",
5721                 "axisPlacement": "auto",
5722                 "barAlignment": 0,
5723                 "drawStyle": "line",
5724                 "fillOpacity": 10,
5725                 "gradientMode": "none",
5726                 "hideFrom": {
5727                   "legend": false,
5728                   "tooltip": false,
5729                   "viz": false
5730                 },
5731                 "lineInterpolation": "linear",
5732                 "lineWidth": 1,
5733                 "pointSize": 5,
5734                 "scaleDistribution": {
5735                   "type": "linear"
5736                 },
5737                 "showPoints": "never",
5738                 "spanNulls": true,
5739                 "stacking": {
5740                   "group": "A",
5741                   "mode": "none"
5742                 },
5743                 "thresholdsStyle": {
5744                   "mode": "off"
5745                 }
5746               },
5747               "links": [],
5748               "mappings": [],
5749               "thresholds": {
5750                 "mode": "absolute",
5751                 "steps": [
5752                   {
5753                     "color": "green"
5754                   },
5755                   {
5756                     "color": "red",
5757                     "value": 80
5758                   }
5759                 ]
5760               },
5761               "unit": "short"
5762             },
5763             "overrides": []
5764           },
5765           "gridPos": {
5766             "h": 8,
5767             "w": 12,
5768             "x": 12,
5769             "y": 33
5770           },
5771           "id": 118,
5772           "options": {
5773             "legend": {
5774               "calcs": [
5775                 "mean",
5776                 "lastNotNull",
5777                 "max"
5778               ],
5779               "displayMode": "table",
5780               "placement": "bottom",
5781               "showLegend": true
5782             },
5783             "tooltip": {
5784               "mode": "single"
5785             }
5786           },
5787           "pluginVersion": "8.1.3",
5788           "targets": [
5789             {
5790               "datasource": {
5791                 "type": "prometheus",
5792                 "uid": "PBFA97CFB590B2093"
5793               },
5794               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5795               "instant": false,
5796               "interval": "",
5797               "legendFormat": "stable",
5798               "refId": "A"
5799             },
5800             {
5801               "datasource": {
5802                 "type": "prometheus",
5803                 "uid": "PBFA97CFB590B2093"
5804               },
5805               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5806               "interval": "",
5807               "legendFormat": "preparing-rebalance",
5808               "refId": "B"
5809             },
5810             {
5811               "datasource": {
5812                 "type": "prometheus",
5813                 "uid": "PBFA97CFB590B2093"
5814               },
5815               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5816               "interval": "",
5817               "legendFormat": "dead",
5818               "refId": "C"
5819             },
5820             {
5821               "datasource": {
5822                 "type": "prometheus",
5823                 "uid": "PBFA97CFB590B2093"
5824               },
5825               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5826               "interval": "",
5827               "legendFormat": "completing-rebalance",
5828               "refId": "D"
5829             },
5830             {
5831               "datasource": {
5832                 "type": "prometheus",
5833                 "uid": "PBFA97CFB590B2093"
5834               },
5835               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5836               "interval": "",
5837               "legendFormat": "empty",
5838               "refId": "E"
5839             }
5840           ],
5841           "title": "Nb consumer groups per state",
5842           "type": "timeseries"
5843         }
5844       ],
5845       "targets": [
5846         {
5847           "datasource": {
5848             "type": "datasource",
5849             "uid": "grafana"
5850           },
5851           "refId": "A"
5852         }
5853       ],
5854       "title": "Group Coordinator",
5855       "type": "row"
5856     },
5857     {
5858       "collapsed": true,
5859       "datasource": {
5860         "type": "datasource",
5861         "uid": "grafana"
5862       },
5863       "gridPos": {
5864         "h": 1,
5865         "w": 24,
5866         "x": 0,
5867         "y": 73
5868       },
5869       "id": 46,
5870       "panels": [
5871         {
5872           "datasource": {
5873             "type": "prometheus",
5874             "uid": "PBFA97CFB590B2093"
5875           },
5876           "description": "The number of messages produced converted to match the log.message.format.version.",
5877           "fieldConfig": {
5878             "defaults": {
5879               "color": {
5880                 "mode": "palette-classic"
5881               },
5882               "custom": {
5883                 "axisLabel": "",
5884                 "axisPlacement": "auto",
5885                 "barAlignment": 0,
5886                 "drawStyle": "line",
5887                 "fillOpacity": 10,
5888                 "gradientMode": "none",
5889                 "hideFrom": {
5890                   "legend": false,
5891                   "tooltip": false,
5892                   "viz": false
5893                 },
5894                 "lineInterpolation": "linear",
5895                 "lineWidth": 1,
5896                 "pointSize": 5,
5897                 "scaleDistribution": {
5898                   "type": "linear"
5899                 },
5900                 "showPoints": "never",
5901                 "spanNulls": true,
5902                 "stacking": {
5903                   "group": "A",
5904                   "mode": "none"
5905                 },
5906                 "thresholdsStyle": {
5907                   "mode": "off"
5908                 }
5909               },
5910               "links": [],
5911               "mappings": [],
5912               "min": 0,
5913               "thresholds": {
5914                 "mode": "absolute",
5915                 "steps": [
5916                   {
5917                     "color": "green"
5918                   },
5919                   {
5920                     "color": "red",
5921                     "value": 80
5922                   }
5923                 ]
5924               },
5925               "unit": "ops"
5926             },
5927             "overrides": []
5928           },
5929           "gridPos": {
5930             "h": 8,
5931             "w": 8,
5932             "x": 0,
5933             "y": 34
5934           },
5935           "id": 48,
5936           "options": {
5937             "legend": {
5938               "calcs": [
5939                 "mean",
5940                 "lastNotNull",
5941                 "max"
5942               ],
5943               "displayMode": "list",
5944               "placement": "bottom",
5945               "showLegend": true
5946             },
5947             "tooltip": {
5948               "mode": "single"
5949             }
5950           },
5951           "pluginVersion": "8.1.3",
5952           "targets": [
5953             {
5954               "datasource": {
5955                 "type": "prometheus",
5956                 "uid": "PBFA97CFB590B2093"
5957               },
5958               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5959               "hide": false,
5960               "interval": "",
5961               "legendFormat": "{{topic}}",
5962               "refId": "A"
5963             }
5964           ],
5965           "title": "Number of produced message conversion",
5966           "type": "timeseries"
5967         },
5968         {
5969           "datasource": {
5970             "type": "prometheus",
5971             "uid": "PBFA97CFB590B2093"
5972           },
5973           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5974           "fieldConfig": {
5975             "defaults": {
5976               "color": {
5977                 "mode": "palette-classic"
5978               },
5979               "custom": {
5980                 "axisLabel": "",
5981                 "axisPlacement": "auto",
5982                 "barAlignment": 0,
5983                 "drawStyle": "line",
5984                 "fillOpacity": 10,
5985                 "gradientMode": "none",
5986                 "hideFrom": {
5987                   "legend": false,
5988                   "tooltip": false,
5989                   "viz": false
5990                 },
5991                 "lineInterpolation": "linear",
5992                 "lineWidth": 1,
5993                 "pointSize": 5,
5994                 "scaleDistribution": {
5995                   "type": "linear"
5996                 },
5997                 "showPoints": "never",
5998                 "spanNulls": true,
5999                 "stacking": {
6000                   "group": "A",
6001                   "mode": "none"
6002                 },
6003                 "thresholdsStyle": {
6004                   "mode": "off"
6005                 }
6006               },
6007               "links": [],
6008               "mappings": [],
6009               "min": 0,
6010               "thresholds": {
6011                 "mode": "absolute",
6012                 "steps": [
6013                   {
6014                     "color": "green"
6015                   },
6016                   {
6017                     "color": "red",
6018                     "value": 80
6019                   }
6020                 ]
6021               },
6022               "unit": "ops"
6023             },
6024             "overrides": []
6025           },
6026           "gridPos": {
6027             "h": 8,
6028             "w": 8,
6029             "x": 8,
6030             "y": 34
6031           },
6032           "id": 51,
6033           "options": {
6034             "legend": {
6035               "calcs": [
6036                 "mean",
6037                 "lastNotNull",
6038                 "max"
6039               ],
6040               "displayMode": "list",
6041               "placement": "bottom",
6042               "showLegend": true
6043             },
6044             "tooltip": {
6045               "mode": "single"
6046             }
6047           },
6048           "pluginVersion": "8.1.3",
6049           "targets": [
6050             {
6051               "datasource": {
6052                 "type": "prometheus",
6053                 "uid": "PBFA97CFB590B2093"
6054               },
6055               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6056               "hide": false,
6057               "interval": "",
6058               "legendFormat": "{{topic}}",
6059               "refId": "A"
6060             }
6061           ],
6062           "title": "Number of consumed message conversion",
6063           "type": "timeseries"
6064         },
6065         {
6066           "datasource": {
6067             "type": "datasource",
6068             "uid": "grafana"
6069           },
6070           "description": "Number of connection per client version",
6071           "fieldConfig": {
6072             "defaults": {
6073               "color": {
6074                 "mode": "palette-classic"
6075               },
6076               "custom": {
6077                 "hideFrom": {
6078                   "legend": false,
6079                   "tooltip": false,
6080                   "viz": false
6081                 }
6082               },
6083               "decimals": 0,
6084               "mappings": [],
6085               "unit": "short"
6086             },
6087             "overrides": []
6088           },
6089           "gridPos": {
6090             "h": 8,
6091             "w": 8,
6092             "x": 16,
6093             "y": 34
6094           },
6095           "id": 96,
6096           "links": [],
6097           "options": {
6098             "legend": {
6099               "calcs": [],
6100               "displayMode": "table",
6101               "placement": "bottom",
6102               "showLegend": true,
6103               "values": [
6104                 "value"
6105               ]
6106             },
6107             "pieType": "pie",
6108             "reduceOptions": {
6109               "calcs": [
6110                 "lastNotNull"
6111               ],
6112               "fields": "",
6113               "values": false
6114             },
6115             "tooltip": {
6116               "mode": "single"
6117             }
6118           },
6119           "targets": [
6120             {
6121               "datasource": {
6122                 "type": "datasource",
6123                 "uid": "grafana"
6124               },
6125               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6126               "interval": "",
6127               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6128               "refId": "A"
6129             }
6130           ],
6131           "title": "Client version repartition",
6132           "type": "piechart"
6133         }
6134       ],
6135       "targets": [
6136         {
6137           "datasource": {
6138             "type": "datasource",
6139             "uid": "grafana"
6140           },
6141           "refId": "A"
6142         }
6143       ],
6144       "title": "Message Conversion",
6145       "type": "row"
6146     }
6147   ],
6148   "refresh": "",
6149   "schemaVersion": 38,
6150   "style": "dark",
6151   "tags": [],
6152   "templating": {
6153     "list": [
6154       {
6155         "current": {
6156           "selected": false,
6157           "text": "dev",
6158           "value": "dev"
6159         },
6160         "datasource": {
6161           "type": "prometheus",
6162           "uid": "PBFA97CFB590B2093"
6163         },
6164         "definition": "label_values(env)",
6165         "hide": 0,
6166         "includeAll": false,
6167         "label": "Environment",
6168         "multi": false,
6169         "name": "env",
6170         "options": [],
6171         "query": {
6172           "query": "label_values(env)",
6173           "refId": "Prometheus-env-Variable-Query"
6174         },
6175         "refresh": 1,
6176         "regex": "",
6177         "skipUrlSync": false,
6178         "sort": 0,
6179         "tagValuesQuery": "",
6180         "tagsQuery": "",
6181         "type": "query",
6182         "useTags": false
6183       },
6184       {
6185         "current": {
6186           "selected": true,
6187           "text": [
6188             "All"
6189           ],
6190           "value": [
6191             "$__all"
6192           ]
6193         },
6194         "datasource": {
6195           "type": "prometheus",
6196           "uid": "PBFA97CFB590B2093"
6197         },
6198         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6199         "hide": 0,
6200         "includeAll": true,
6201         "label": "Instance",
6202         "multi": true,
6203         "name": "instance",
6204         "options": [],
6205         "query": {
6206           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6207           "refId": "Prometheus-instance-Variable-Query"
6208         },
6209         "refresh": 1,
6210         "regex": "",
6211         "skipUrlSync": false,
6212         "sort": 0,
6213         "tagValuesQuery": "",
6214         "tagsQuery": "",
6215         "type": "query",
6216         "useTags": false
6217       },
6218       {
6219         "allValue": "",
6220         "current": {
6221           "selected": true,
6222           "text": [
6223             "0.95"
6224           ],
6225           "value": [
6226             "0.95"
6227           ]
6228         },
6229         "datasource": {
6230           "type": "prometheus",
6231           "uid": "PBFA97CFB590B2093"
6232         },
6233         "definition": "label_values(quantile)",
6234         "hide": 0,
6235         "includeAll": true,
6236         "label": "Percentile",
6237         "multi": true,
6238         "name": "percentile",
6239         "options": [],
6240         "query": {
6241           "query": "label_values(quantile)",
6242           "refId": "Prometheus-percentile-Variable-Query"
6243         },
6244         "refresh": 1,
6245         "regex": "",
6246         "skipUrlSync": false,
6247         "sort": 0,
6248         "tagValuesQuery": "",
6249         "tagsQuery": "",
6250         "type": "query",
6251         "useTags": false
6252       }
6253     ]
6254   },
6255   "time": {
6256     "from": "now-5m",
6257     "to": "now"
6258   },
6259   "timepicker": {
6260     "refresh_intervals": [
6261       "5s",
6262       "10s",
6263       "30s",
6264       "1m",
6265       "5m",
6266       "15m",
6267       "30m",
6268       "1h",
6269       "2h",
6270       "1d"
6271     ],
6272     "time_options": [
6273       "5m",
6274       "15m",
6275       "1h",
6276       "6h",
6277       "12h",
6278       "24h",
6279       "2d",
6280       "7d",
6281       "30d"
6282     ]
6283   },
6284   "timezone": "browser",
6285   "title": "Kafka cluster",
6286   "uid": "qu-QZdfZz",
6287   "version": 1,
6288   "weekStart": ""
6289 }