juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of active controllers in the cluster.",
  64       "fieldConfig": {
  65         "defaults": {
  66           "mappings": [
  67             {
  68               "options": {
  69                 "match": "null",
  70                 "result": {
  71                   "text": "N/A"
  72                 }
  73               },
  74               "type": "special"
  75             }
  76           ],
  77           "thresholds": {
  78             "mode": "absolute",
  79             "steps": [
  80               {
  81                 "color": "#299c46",
  82                 "value": null
  83               },
  84               {
  85                 "color": "#e5ac0e",
  86                 "value": 2
  87               },
  88               {
  89                 "color": "#bf1b00"
  90               }
  91             ]
  92           },
  93           "unit": "none"
  94         },
  95         "overrides": []
  96       },
  97       "gridPos": {
  98         "h": 4,
  99         "w": 4,
 100         "x": 0,
 101         "y": 1
 102       },
 103       "id": 12,
 104       "links": [],
 105       "maxDataPoints": 100,
 106       "options": {
 107         "colorMode": "value",
 108         "graphMode": "none",
 109         "justifyMode": "auto",
 110         "orientation": "vertical",
 111         "reduceOptions": {
 112           "calcs": [
 113             "lastNotNull"
 114           ],
 115           "fields": "",
 116           "values": false
 117         },
 118         "text": {},
 119         "textMode": "value_and_name"
 120       },
 121       "pluginVersion": "10.0.3",
 122       "targets": [
 123         {
 124           "datasource": {
 125             "type": "prometheus",
 126             "uid": "PBFA97CFB590B2093"
 127           },
 128           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 129           "format": "time_series",
 130           "instant": true,
 131           "interval": "",
 132           "intervalFactor": 1,
 133           "legendFormat": "{{instance}}",
 134           "refId": "A"
 135         }
 136       ],
 137       "title": "Active Controllers",
 138       "type": "stat"
 139     },
 140     {
 141       "datasource": {
 142         "type": "prometheus",
 143         "uid": "PBFA97CFB590B2093"
 144       },
 145       "description": "Number of Brokers Online",
 146       "fieldConfig": {
 147         "defaults": {
 148           "color": {
 149             "mode": "thresholds"
 150           },
 151           "mappings": [
 152             {
 153               "options": {
 154                 "match": "null",
 155                 "result": {
 156                   "text": "N/A"
 157                 }
 158               },
 159               "type": "special"
 160             }
 161           ],
 162           "thresholds": {
 163             "mode": "absolute",
 164             "steps": [
 165               {
 166                 "color": "#d44a3a",
 167                 "value": null
 168               },
 169               {
 170                 "color": "rgba(237, 129, 40, 0.89)",
 171                 "value": 0
 172               },
 173               {
 174                 "color": "#299c46",
 175                 "value": 2
 176               }
 177             ]
 178           },
 179           "unit": "none"
 180         },
 181         "overrides": []
 182       },
 183       "gridPos": {
 184         "h": 4,
 185         "w": 4,
 186         "x": 4,
 187         "y": 1
 188       },
 189       "id": 14,
 190       "links": [],
 191       "maxDataPoints": 100,
 192       "options": {
 193         "colorMode": "value",
 194         "graphMode": "none",
 195         "justifyMode": "auto",
 196         "orientation": "horizontal",
 197         "reduceOptions": {
 198           "calcs": [
 199             "lastNotNull"
 200           ],
 201           "fields": "",
 202           "values": false
 203         },
 204         "text": {},
 205         "textMode": "auto"
 206       },
 207       "pluginVersion": "10.0.3",
 208       "repeatDirection": "h",
 209       "targets": [
 210         {
 211           "datasource": {
 212             "type": "prometheus",
 213             "uid": "PBFA97CFB590B2093"
 214           },
 215           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 216           "format": "time_series",
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 1,
 220           "legendFormat": "",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Brokers Online",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Unclean leader election rate",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#299c46",
 254                 "value": null
 255               },
 256               {
 257                 "color": "rgba(237, 129, 40, 0.89)",
 258                 "value": 2
 259               },
 260               {
 261                 "color": "#d44a3a"
 262               }
 263             ]
 264           },
 265           "unit": "none"
 266         },
 267         "overrides": []
 268       },
 269       "gridPos": {
 270         "h": 4,
 271         "w": 4,
 272         "x": 11,
 273         "y": 1
 274       },
 275       "id": 16,
 276       "links": [],
 277       "maxDataPoints": 100,
 278       "options": {
 279         "colorMode": "value",
 280         "graphMode": "none",
 281         "justifyMode": "auto",
 282         "orientation": "horizontal",
 283         "reduceOptions": {
 284           "calcs": [
 285             "lastNotNull"
 286           ],
 287           "fields": "",
 288           "values": false
 289         },
 290         "text": {},
 291         "textMode": "auto"
 292       },
 293       "pluginVersion": "10.0.3",
 294       "targets": [
 295         {
 296           "datasource": {
 297             "type": "prometheus",
 298             "uid": "PBFA97CFB590B2093"
 299           },
 300           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 301           "format": "time_series",
 302           "instant": true,
 303           "interval": "",
 304           "intervalFactor": 1,
 305           "legendFormat": "",
 306           "refId": "A"
 307         }
 308       ],
 309       "title": "Unclean Leader Election Rate",
 310       "type": "stat"
 311     },
 312     {
 313       "datasource": {
 314         "type": "prometheus",
 315         "uid": "PBFA97CFB590B2093"
 316       },
 317       "description": "Partitions that are online",
 318       "fieldConfig": {
 319         "defaults": {
 320           "color": {
 321             "mode": "thresholds"
 322           },
 323           "mappings": [
 324             {
 325               "options": {
 326                 "match": "null",
 327                 "result": {
 328                   "text": "N/A"
 329                 }
 330               },
 331               "type": "special"
 332             }
 333           ],
 334           "thresholds": {
 335             "mode": "absolute",
 336             "steps": [
 337               {
 338                 "color": "#d44a3a",
 339                 "value": null
 340               },
 341               {
 342                 "color": "rgba(237, 129, 40, 0.89)",
 343                 "value": 0
 344               },
 345               {
 346                 "color": "#299c46",
 347                 "value": 0
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 15,
 359         "y": 1
 360       },
 361       "id": 18,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Online Partitions",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#299c46",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 2
 430               },
 431               {
 432                 "color": "#d44a3a"
 433               }
 434             ]
 435           },
 436           "unit": "none"
 437         },
 438         "overrides": []
 439       },
 440       "gridPos": {
 441         "h": 4,
 442         "w": 4,
 443         "x": 19,
 444         "y": 1
 445       },
 446       "id": 33,
 447       "links": [],
 448       "maxDataPoints": 100,
 449       "options": {
 450         "colorMode": "value",
 451         "graphMode": "none",
 452         "justifyMode": "auto",
 453         "orientation": "horizontal",
 454         "reduceOptions": {
 455           "calcs": [
 456             "lastNotNull"
 457           ],
 458           "fields": "",
 459           "values": false
 460         },
 461         "text": {},
 462         "textMode": "auto"
 463       },
 464       "pluginVersion": "10.0.3",
 465       "targets": [
 466         {
 467           "datasource": {
 468             "type": "prometheus",
 469             "uid": "PBFA97CFB590B2093"
 470           },
 471           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 472           "format": "time_series",
 473           "instant": true,
 474           "interval": "",
 475           "intervalFactor": 1,
 476           "legendFormat": "",
 477           "refId": "A"
 478         }
 479       ],
 480       "title": "Preferred Replica Imbalance",
 481       "type": "stat"
 482     },
 483     {
 484       "datasource": {
 485         "type": "prometheus",
 486         "uid": "PBFA97CFB590B2093"
 487       },
 488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 489       "fieldConfig": {
 490         "defaults": {
 491           "color": {
 492             "mode": "thresholds"
 493           },
 494           "mappings": [
 495             {
 496               "options": {
 497                 "match": "null",
 498                 "result": {
 499                   "text": "N/A"
 500                 }
 501               },
 502               "type": "special"
 503             }
 504           ],
 505           "thresholds": {
 506             "mode": "absolute",
 507             "steps": [
 508               {
 509                 "color": "#508642",
 510                 "value": null
 511               },
 512               {
 513                 "color": "rgba(237, 129, 40, 0.89)",
 514                 "value": 1
 515               },
 516               {
 517                 "color": "#bf1b00",
 518                 "value": 5
 519               }
 520             ]
 521           },
 522           "unit": "none"
 523         },
 524         "overrides": []
 525       },
 526       "gridPos": {
 527         "h": 4,
 528         "w": 4,
 529         "x": 0,
 530         "y": 5
 531       },
 532       "id": 20,
 533       "links": [],
 534       "maxDataPoints": 100,
 535       "options": {
 536         "colorMode": "value",
 537         "graphMode": "none",
 538         "justifyMode": "auto",
 539         "orientation": "horizontal",
 540         "reduceOptions": {
 541           "calcs": [
 542             "lastNotNull"
 543           ],
 544           "fields": "",
 545           "values": false
 546         },
 547         "text": {},
 548         "textMode": "auto"
 549       },
 550       "pluginVersion": "10.0.3",
 551       "targets": [
 552         {
 553           "datasource": {
 554             "type": "prometheus",
 555             "uid": "PBFA97CFB590B2093"
 556           },
 557           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 558           "format": "time_series",
 559           "hide": false,
 560           "instant": true,
 561           "interval": "",
 562           "intervalFactor": 2,
 563           "legendFormat": "",
 564           "refId": "A"
 565         }
 566       ],
 567       "title": "Under Replicated Partitions",
 568       "type": "stat"
 569     },
 570     {
 571       "datasource": {
 572         "type": "prometheus",
 573         "uid": "PBFA97CFB590B2093"
 574       },
 575       "description": "",
 576       "fieldConfig": {
 577         "defaults": {
 578           "color": {
 579             "mode": "palette-classic"
 580           },
 581           "custom": {
 582             "axisCenteredZero": false,
 583             "axisColorMode": "text",
 584             "axisLabel": "",
 585             "axisPlacement": "auto",
 586             "barAlignment": 0,
 587             "drawStyle": "line",
 588             "fillOpacity": 10,
 589             "gradientMode": "none",
 590             "hideFrom": {
 591               "legend": false,
 592               "tooltip": false,
 593               "viz": false
 594             },
 595             "lineInterpolation": "linear",
 596             "lineWidth": 1,
 597             "pointSize": 5,
 598             "scaleDistribution": {
 599               "type": "linear"
 600             },
 601             "showPoints": "never",
 602             "spanNulls": true,
 603             "stacking": {
 604               "group": "A",
 605               "mode": "none"
 606             },
 607             "thresholdsStyle": {
 608               "mode": "off"
 609             }
 610           },
 611           "decimals": 0,
 612           "links": [],
 613           "mappings": [],
 614           "min": 0,
 615           "thresholds": {
 616             "mode": "absolute",
 617             "steps": [
 618               {
 619                 "color": "green",
 620                 "value": null
 621               },
 622               {
 623                 "color": "red",
 624                 "value": 80
 625               }
 626             ]
 627           },
 628           "unit": "partitions"
 629         },
 630         "overrides": []
 631       },
 632       "gridPos": {
 633         "h": 12,
 634         "w": 11,
 635         "x": 4,
 636         "y": 5
 637       },
 638       "id": 123,
 639       "options": {
 640         "legend": {
 641           "calcs": [
 642             "lastNotNull",
 643             "max"
 644           ],
 645           "displayMode": "table",
 646           "placement": "bottom",
 647           "showLegend": true
 648         },
 649         "tooltip": {
 650           "mode": "single",
 651           "sort": "none"
 652         }
 653       },
 654       "pluginVersion": "8.1.3",
 655       "targets": [
 656         {
 657           "datasource": {
 658             "type": "prometheus",
 659             "uid": "PBFA97CFB590B2093"
 660           },
 661           "editorMode": "code",
 662           "exemplar": false,
 663           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 664           "hide": false,
 665           "interval": "",
 666           "legendFormat": "{{instance}}",
 667           "range": true,
 668           "refId": "A"
 669         }
 670       ],
 671       "title": "Underreplicated Partitions",
 672       "type": "timeseries"
 673     },
 674     {
 675       "datasource": {
 676         "type": "prometheus",
 677         "uid": "PBFA97CFB590B2093"
 678       },
 679       "description": "",
 680       "fieldConfig": {
 681         "defaults": {
 682           "color": {
 683             "mode": "palette-classic"
 684           },
 685           "custom": {
 686             "axisCenteredZero": false,
 687             "axisColorMode": "text",
 688             "axisLabel": "",
 689             "axisPlacement": "auto",
 690             "barAlignment": 0,
 691             "drawStyle": "line",
 692             "fillOpacity": 10,
 693             "gradientMode": "none",
 694             "hideFrom": {
 695               "legend": false,
 696               "tooltip": false,
 697               "viz": false
 698             },
 699             "lineInterpolation": "linear",
 700             "lineWidth": 1,
 701             "pointSize": 5,
 702             "scaleDistribution": {
 703               "type": "linear"
 704             },
 705             "showPoints": "never",
 706             "spanNulls": true,
 707             "stacking": {
 708               "group": "A",
 709               "mode": "none"
 710             },
 711             "thresholdsStyle": {
 712               "mode": "off"
 713             }
 714           },
 715           "decimals": 0,
 716           "links": [],
 717           "mappings": [],
 718           "min": 0,
 719           "thresholds": {
 720             "mode": "absolute",
 721             "steps": [
 722               {
 723                 "color": "green",
 724                 "value": null
 725               },
 726               {
 727                 "color": "red",
 728                 "value": 80
 729               }
 730             ]
 731           },
 732           "unit": "partitions"
 733         },
 734         "overrides": []
 735       },
 736       "gridPos": {
 737         "h": 12,
 738         "w": 8,
 739         "x": 15,
 740         "y": 5
 741       },
 742       "id": 124,
 743       "options": {
 744         "legend": {
 745           "calcs": [
 746             "lastNotNull",
 747             "max"
 748           ],
 749           "displayMode": "table",
 750           "placement": "bottom",
 751           "showLegend": true
 752         },
 753         "tooltip": {
 754           "mode": "single",
 755           "sort": "none"
 756         }
 757       },
 758       "pluginVersion": "8.1.3",
 759       "targets": [
 760         {
 761           "datasource": {
 762             "type": "prometheus",
 763             "uid": "PBFA97CFB590B2093"
 764           },
 765           "editorMode": "code",
 766           "exemplar": false,
 767           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 768           "hide": false,
 769           "interval": "",
 770           "legendFormat": "{{instance}}",
 771           "range": true,
 772           "refId": "A"
 773         }
 774       ],
 775       "title": "Leader Count",
 776       "type": "timeseries"
 777     },
 778     {
 779       "datasource": {
 780         "type": "prometheus",
 781         "uid": "PBFA97CFB590B2093"
 782       },
 783       "description": "Number of partitions under min insync replicas.",
 784       "fieldConfig": {
 785         "defaults": {
 786           "color": {
 787             "mode": "thresholds"
 788           },
 789           "mappings": [
 790             {
 791               "options": {
 792                 "match": "null",
 793                 "result": {
 794                   "text": "N/A"
 795                 }
 796               },
 797               "type": "special"
 798             }
 799           ],
 800           "thresholds": {
 801             "mode": "absolute",
 802             "steps": [
 803               {
 804                 "color": "#508642",
 805                 "value": null
 806               },
 807               {
 808                 "color": "rgba(237, 129, 40, 0.89)",
 809                 "value": 1
 810               },
 811               {
 812                 "color": "#bf1b00",
 813                 "value": 5
 814               }
 815             ]
 816           },
 817           "unit": "none"
 818         },
 819         "overrides": []
 820       },
 821       "gridPos": {
 822         "h": 4,
 823         "w": 4,
 824         "x": 0,
 825         "y": 9
 826       },
 827       "id": 32,
 828       "links": [],
 829       "maxDataPoints": 100,
 830       "options": {
 831         "colorMode": "value",
 832         "graphMode": "none",
 833         "justifyMode": "auto",
 834         "orientation": "horizontal",
 835         "reduceOptions": {
 836           "calcs": [
 837             "lastNotNull"
 838           ],
 839           "fields": "",
 840           "values": false
 841         },
 842         "text": {},
 843         "textMode": "auto"
 844       },
 845       "pluginVersion": "10.0.3",
 846       "targets": [
 847         {
 848           "datasource": {
 849             "type": "prometheus",
 850             "uid": "PBFA97CFB590B2093"
 851           },
 852           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 853           "format": "time_series",
 854           "hide": false,
 855           "instant": true,
 856           "interval": "",
 857           "intervalFactor": 2,
 858           "legendFormat": "",
 859           "refId": "A"
 860         }
 861       ],
 862       "title": "Under Min ISR Partitions",
 863       "type": "stat"
 864     },
 865     {
 866       "datasource": {
 867         "type": "prometheus",
 868         "uid": "PBFA97CFB590B2093"
 869       },
 870       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 871       "fieldConfig": {
 872         "defaults": {
 873           "color": {
 874             "mode": "thresholds"
 875           },
 876           "mappings": [
 877             {
 878               "options": {
 879                 "match": "null",
 880                 "result": {
 881                   "text": "N/A"
 882                 }
 883               },
 884               "type": "special"
 885             }
 886           ],
 887           "thresholds": {
 888             "mode": "absolute",
 889             "steps": [
 890               {
 891                 "color": "#508642",
 892                 "value": null
 893               },
 894               {
 895                 "color": "#ef843c",
 896                 "value": 1
 897               },
 898               {
 899                 "color": "#bf1b00",
 900                 "value": 1
 901               }
 902             ]
 903           },
 904           "unit": "none"
 905         },
 906         "overrides": []
 907       },
 908       "gridPos": {
 909         "h": 4,
 910         "w": 4,
 911         "x": 0,
 912         "y": 13
 913       },
 914       "id": 22,
 915       "links": [],
 916       "maxDataPoints": 100,
 917       "options": {
 918         "colorMode": "value",
 919         "graphMode": "none",
 920         "justifyMode": "auto",
 921         "orientation": "horizontal",
 922         "reduceOptions": {
 923           "calcs": [
 924             "lastNotNull"
 925           ],
 926           "fields": "",
 927           "values": false
 928         },
 929         "text": {},
 930         "textMode": "auto"
 931       },
 932       "pluginVersion": "10.0.3",
 933       "targets": [
 934         {
 935           "datasource": {
 936             "type": "prometheus",
 937             "uid": "PBFA97CFB590B2093"
 938           },
 939           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 940           "format": "time_series",
 941           "instant": true,
 942           "interval": "",
 943           "intervalFactor": 1,
 944           "legendFormat": "",
 945           "refId": "A"
 946         }
 947       ],
 948       "title": "Offline Partitions Count",
 949       "type": "stat"
 950     },
 951     {
 952       "collapsed": false,
 953       "datasource": {
 954         "type": "datasource",
 955         "uid": "grafana"
 956       },
 957       "gridPos": {
 958         "h": 1,
 959         "w": 24,
 960         "x": 0,
 961         "y": 17
 962       },
 963       "id": 31,
 964       "panels": [],
 965       "targets": [
 966         {
 967           "datasource": {
 968             "type": "datasource",
 969             "uid": "grafana"
 970           },
 971           "refId": "A"
 972         }
 973       ],
 974       "title": "Throughput & Request rate",
 975       "type": "row"
 976     },
 977     {
 978       "datasource": {
 979         "type": "prometheus",
 980         "uid": "PBFA97CFB590B2093"
 981       },
 982       "fieldConfig": {
 983         "defaults": {
 984           "color": {
 985             "mode": "palette-classic"
 986           },
 987           "custom": {
 988             "axisCenteredZero": false,
 989             "axisColorMode": "text",
 990             "axisLabel": "Bytes/s",
 991             "axisPlacement": "auto",
 992             "barAlignment": 0,
 993             "drawStyle": "line",
 994             "fillOpacity": 10,
 995             "gradientMode": "none",
 996             "hideFrom": {
 997               "legend": false,
 998               "tooltip": false,
 999               "viz": false
1000             },
1001             "lineInterpolation": "linear",
1002             "lineWidth": 2,
1003             "pointSize": 5,
1004             "scaleDistribution": {
1005               "type": "linear"
1006             },
1007             "showPoints": "never",
1008             "spanNulls": false,
1009             "stacking": {
1010               "group": "A",
1011               "mode": "none"
1012             },
1013             "thresholdsStyle": {
1014               "mode": "off"
1015             }
1016           },
1017           "links": [],
1018           "mappings": [],
1019           "thresholds": {
1020             "mode": "absolute",
1021             "steps": [
1022               {
1023                 "color": "green",
1024                 "value": null
1025               },
1026               {
1027                 "color": "red",
1028                 "value": 80
1029               }
1030             ]
1031           },
1032           "unit": "Bps"
1033         },
1034         "overrides": []
1035       },
1036       "gridPos": {
1037         "h": 8,
1038         "w": 12,
1039         "x": 0,
1040         "y": 18
1041       },
1042       "id": 84,
1043       "links": [],
1044       "options": {
1045         "legend": {
1046           "calcs": [
1047             "mean",
1048             "lastNotNull",
1049             "max"
1050           ],
1051           "displayMode": "list",
1052           "placement": "bottom",
1053           "showLegend": true
1054         },
1055         "tooltip": {
1056           "mode": "single",
1057           "sort": "none"
1058         }
1059       },
1060       "pluginVersion": "8.1.3",
1061       "targets": [
1062         {
1063           "datasource": {
1064             "type": "prometheus",
1065             "uid": "PBFA97CFB590B2093"
1066           },
1067           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1068           "format": "time_series",
1069           "interval": "",
1070           "intervalFactor": 2,
1071           "legendFormat": "Bytes in",
1072           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1073           "refId": "A",
1074           "step": 4
1075         },
1076         {
1077           "datasource": {
1078             "type": "prometheus",
1079             "uid": "PBFA97CFB590B2093"
1080           },
1081           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1082           "format": "time_series",
1083           "hide": false,
1084           "interval": "",
1085           "intervalFactor": 2,
1086           "legendFormat": "Bytes out",
1087           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1088           "refId": "B",
1089           "step": 4
1090         }
1091       ],
1092       "title": "Broker network throughput",
1093       "type": "timeseries"
1094     },
1095     {
1096       "datasource": {
1097         "type": "prometheus",
1098         "uid": "PBFA97CFB590B2093"
1099       },
1100       "description": "Produce request rate.",
1101       "fieldConfig": {
1102         "defaults": {
1103           "mappings": [
1104             {
1105               "options": {
1106                 "match": "null",
1107                 "result": {
1108                   "text": "N/A"
1109                 }
1110               },
1111               "type": "special"
1112             }
1113           ],
1114           "thresholds": {
1115             "mode": "absolute",
1116             "steps": [
1117               {
1118                 "color": "green",
1119                 "value": null
1120               }
1121             ]
1122           },
1123           "unit": "none"
1124         },
1125         "overrides": []
1126       },
1127       "gridPos": {
1128         "h": 4,
1129         "w": 4,
1130         "x": 12,
1131         "y": 18
1132       },
1133       "id": 93,
1134       "links": [],
1135       "maxDataPoints": 100,
1136       "options": {
1137         "colorMode": "value",
1138         "graphMode": "area",
1139         "justifyMode": "auto",
1140         "orientation": "horizontal",
1141         "reduceOptions": {
1142           "calcs": [
1143             "lastNotNull"
1144           ],
1145           "fields": "",
1146           "values": false
1147         },
1148         "text": {},
1149         "textMode": "auto"
1150       },
1151       "pluginVersion": "10.0.3",
1152       "targets": [
1153         {
1154           "datasource": {
1155             "type": "prometheus",
1156             "uid": "PBFA97CFB590B2093"
1157           },
1158           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1159           "interval": "",
1160           "legendFormat": "",
1161           "refId": "A"
1162         }
1163       ],
1164       "title": "All Request Per Sec",
1165       "type": "stat"
1166     },
1167     {
1168       "datasource": {
1169         "type": "prometheus",
1170         "uid": "PBFA97CFB590B2093"
1171       },
1172       "description": "Fetch request rate.",
1173       "fieldConfig": {
1174         "defaults": {
1175           "mappings": [
1176             {
1177               "options": {
1178                 "match": "null",
1179                 "result": {
1180                   "text": "N/A"
1181                 }
1182               },
1183               "type": "special"
1184             }
1185           ],
1186           "thresholds": {
1187             "mode": "absolute",
1188             "steps": [
1189               {
1190                 "color": "green",
1191                 "value": null
1192               }
1193             ]
1194           },
1195           "unit": "none"
1196         },
1197         "overrides": []
1198       },
1199       "gridPos": {
1200         "h": 4,
1201         "w": 4,
1202         "x": 16,
1203         "y": 18
1204       },
1205       "id": 37,
1206       "links": [],
1207       "maxDataPoints": 100,
1208       "options": {
1209         "colorMode": "value",
1210         "graphMode": "area",
1211         "justifyMode": "auto",
1212         "orientation": "horizontal",
1213         "reduceOptions": {
1214           "calcs": [
1215             "lastNotNull"
1216           ],
1217           "fields": "",
1218           "values": false
1219         },
1220         "text": {},
1221         "textMode": "auto"
1222       },
1223       "pluginVersion": "10.0.3",
1224       "targets": [
1225         {
1226           "datasource": {
1227             "type": "prometheus",
1228             "uid": "PBFA97CFB590B2093"
1229           },
1230           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1231           "interval": "",
1232           "legendFormat": "",
1233           "refId": "A"
1234         }
1235       ],
1236       "title": "Consumer Fetch Request Per Sec",
1237       "type": "stat"
1238     },
1239     {
1240       "datasource": {
1241         "type": "prometheus",
1242         "uid": "PBFA97CFB590B2093"
1243       },
1244       "description": "Produce request rate.",
1245       "fieldConfig": {
1246         "defaults": {
1247           "mappings": [
1248             {
1249               "options": {
1250                 "match": "null",
1251                 "result": {
1252                   "text": "N/A"
1253                 }
1254               },
1255               "type": "special"
1256             }
1257           ],
1258           "thresholds": {
1259             "mode": "absolute",
1260             "steps": [
1261               {
1262                 "color": "green",
1263                 "value": null
1264               }
1265             ]
1266           },
1267           "unit": "none"
1268         },
1269         "overrides": []
1270       },
1271       "gridPos": {
1272         "h": 4,
1273         "w": 4,
1274         "x": 20,
1275         "y": 18
1276       },
1277       "id": 35,
1278       "links": [],
1279       "maxDataPoints": 100,
1280       "options": {
1281         "colorMode": "value",
1282         "graphMode": "area",
1283         "justifyMode": "auto",
1284         "orientation": "horizontal",
1285         "reduceOptions": {
1286           "calcs": [
1287             "lastNotNull"
1288           ],
1289           "fields": "",
1290           "values": false
1291         },
1292         "text": {},
1293         "textMode": "auto"
1294       },
1295       "pluginVersion": "10.0.3",
1296       "targets": [
1297         {
1298           "datasource": {
1299             "type": "prometheus",
1300             "uid": "PBFA97CFB590B2093"
1301           },
1302           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1303           "interval": "",
1304           "legendFormat": "",
1305           "refId": "A"
1306         }
1307       ],
1308       "title": "Produce Request Per Sec",
1309       "type": "stat"
1310     },
1311     {
1312       "datasource": {
1313         "type": "prometheus",
1314         "uid": "PBFA97CFB590B2093"
1315       },
1316       "description": "Metadata request rate.",
1317       "fieldConfig": {
1318         "defaults": {
1319           "mappings": [
1320             {
1321               "options": {
1322                 "match": "null",
1323                 "result": {
1324                   "text": "N/A"
1325                 }
1326               },
1327               "type": "special"
1328             }
1329           ],
1330           "thresholds": {
1331             "mode": "absolute",
1332             "steps": [
1333               {
1334                 "color": "green",
1335                 "value": null
1336               }
1337             ]
1338           },
1339           "unit": "none"
1340         },
1341         "overrides": []
1342       },
1343       "gridPos": {
1344         "h": 4,
1345         "w": 4,
1346         "x": 12,
1347         "y": 22
1348       },
1349       "id": 36,
1350       "links": [],
1351       "maxDataPoints": 100,
1352       "options": {
1353         "colorMode": "value",
1354         "graphMode": "area",
1355         "justifyMode": "auto",
1356         "orientation": "horizontal",
1357         "reduceOptions": {
1358           "calcs": [
1359             "lastNotNull"
1360           ],
1361           "fields": "",
1362           "values": false
1363         },
1364         "text": {},
1365         "textMode": "auto"
1366       },
1367       "pluginVersion": "10.0.3",
1368       "targets": [
1369         {
1370           "datasource": {
1371             "type": "prometheus",
1372             "uid": "PBFA97CFB590B2093"
1373           },
1374           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1375           "interval": "",
1376           "legendFormat": "",
1377           "refId": "A"
1378         }
1379       ],
1380       "title": "Metadata Request Per Sec",
1381       "type": "stat"
1382     },
1383     {
1384       "datasource": {
1385         "type": "prometheus",
1386         "uid": "PBFA97CFB590B2093"
1387       },
1388       "description": "Offset Commit request rate.",
1389       "fieldConfig": {
1390         "defaults": {
1391           "mappings": [
1392             {
1393               "options": {
1394                 "match": "null",
1395                 "result": {
1396                   "text": "N/A"
1397                 }
1398               },
1399               "type": "special"
1400             }
1401           ],
1402           "thresholds": {
1403             "mode": "absolute",
1404             "steps": [
1405               {
1406                 "color": "green",
1407                 "value": null
1408               }
1409             ]
1410           },
1411           "unit": "none"
1412         },
1413         "overrides": []
1414       },
1415       "gridPos": {
1416         "h": 4,
1417         "w": 4,
1418         "x": 16,
1419         "y": 22
1420       },
1421       "id": 38,
1422       "links": [],
1423       "maxDataPoints": 100,
1424       "options": {
1425         "colorMode": "value",
1426         "graphMode": "area",
1427         "justifyMode": "auto",
1428         "orientation": "horizontal",
1429         "reduceOptions": {
1430           "calcs": [
1431             "lastNotNull"
1432           ],
1433           "fields": "",
1434           "values": false
1435         },
1436         "text": {},
1437         "textMode": "auto"
1438       },
1439       "pluginVersion": "10.0.3",
1440       "targets": [
1441         {
1442           "datasource": {
1443             "type": "prometheus",
1444             "uid": "PBFA97CFB590B2093"
1445           },
1446           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1447           "interval": "",
1448           "legendFormat": "",
1449           "refId": "A"
1450         }
1451       ],
1452       "title": "Offset Commit Request Per Sec",
1453       "type": "stat"
1454     },
1455     {
1456       "datasource": {
1457         "type": "prometheus",
1458         "uid": "PBFA97CFB590B2093"
1459       },
1460       "description": "Fetch request rate.",
1461       "fieldConfig": {
1462         "defaults": {
1463           "mappings": [
1464             {
1465               "options": {
1466                 "match": "null",
1467                 "result": {
1468                   "text": "N/A"
1469                 }
1470               },
1471               "type": "special"
1472             }
1473           ],
1474           "thresholds": {
1475             "mode": "absolute",
1476             "steps": [
1477               {
1478                 "color": "green",
1479                 "value": null
1480               }
1481             ]
1482           },
1483           "unit": "none"
1484         },
1485         "overrides": []
1486       },
1487       "gridPos": {
1488         "h": 4,
1489         "w": 4,
1490         "x": 20,
1491         "y": 22
1492       },
1493       "id": 94,
1494       "links": [],
1495       "maxDataPoints": 100,
1496       "options": {
1497         "colorMode": "value",
1498         "graphMode": "area",
1499         "justifyMode": "auto",
1500         "orientation": "horizontal",
1501         "reduceOptions": {
1502           "calcs": [
1503             "lastNotNull"
1504           ],
1505           "fields": "",
1506           "values": false
1507         },
1508         "text": {},
1509         "textMode": "auto"
1510       },
1511       "pluginVersion": "10.0.3",
1512       "targets": [
1513         {
1514           "datasource": {
1515             "type": "prometheus",
1516             "uid": "PBFA97CFB590B2093"
1517           },
1518           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1519           "interval": "",
1520           "legendFormat": "",
1521           "refId": "A"
1522         }
1523       ],
1524       "title": "Broker Fetch Request Per Sec",
1525       "type": "stat"
1526     },
1527     {
1528       "collapsed": false,
1529       "datasource": {
1530         "type": "datasource",
1531         "uid": "grafana"
1532       },
1533       "gridPos": {
1534         "h": 1,
1535         "w": 24,
1536         "x": 0,
1537         "y": 26
1538       },
1539       "id": 40,
1540       "panels": [],
1541       "targets": [
1542         {
1543           "datasource": {
1544             "type": "datasource",
1545             "uid": "grafana"
1546           },
1547           "refId": "A"
1548         }
1549       ],
1550       "title": "System",
1551       "type": "row"
1552     },
1553     {
1554       "datasource": {
1555         "type": "prometheus",
1556         "uid": "PBFA97CFB590B2093"
1557       },
1558       "fieldConfig": {
1559         "defaults": {
1560           "color": {
1561             "mode": "palette-classic"
1562           },
1563           "custom": {
1564             "axisCenteredZero": false,
1565             "axisColorMode": "text",
1566             "axisLabel": "Cores",
1567             "axisPlacement": "auto",
1568             "barAlignment": 0,
1569             "drawStyle": "line",
1570             "fillOpacity": 10,
1571             "gradientMode": "none",
1572             "hideFrom": {
1573               "legend": false,
1574               "tooltip": false,
1575               "viz": false
1576             },
1577             "lineInterpolation": "linear",
1578             "lineWidth": 2,
1579             "pointSize": 5,
1580             "scaleDistribution": {
1581               "type": "linear"
1582             },
1583             "showPoints": "never",
1584             "spanNulls": false,
1585             "stacking": {
1586               "group": "A",
1587               "mode": "none"
1588             },
1589             "thresholdsStyle": {
1590               "mode": "off"
1591             }
1592           },
1593           "links": [],
1594           "mappings": [],
1595           "thresholds": {
1596             "mode": "absolute",
1597             "steps": [
1598               {
1599                 "color": "green",
1600                 "value": null
1601               },
1602               {
1603                 "color": "red",
1604                 "value": 80
1605               }
1606             ]
1607           },
1608           "unit": "percent"
1609         },
1610         "overrides": [
1611           {
1612             "matcher": {
1613               "id": "byName",
1614               "options": "localhost:7071"
1615             },
1616             "properties": [
1617               {
1618                 "id": "color",
1619                 "value": {
1620                   "fixedColor": "#629E51",
1621                   "mode": "fixed"
1622                 }
1623               }
1624             ]
1625           }
1626         ]
1627       },
1628       "gridPos": {
1629         "h": 12,
1630         "w": 8,
1631         "x": 0,
1632         "y": 27
1633       },
1634       "id": 27,
1635       "links": [],
1636       "options": {
1637         "legend": {
1638           "calcs": [
1639             "mean",
1640             "lastNotNull",
1641             "max"
1642           ],
1643           "displayMode": "table",
1644           "placement": "bottom",
1645           "showLegend": true
1646         },
1647         "tooltip": {
1648           "mode": "single",
1649           "sort": "none"
1650         }
1651       },
1652       "pluginVersion": "8.1.3",
1653       "targets": [
1654         {
1655           "datasource": {
1656             "type": "prometheus",
1657             "uid": "PBFA97CFB590B2093"
1658           },
1659           "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1660           "format": "time_series",
1661           "interval": "",
1662           "intervalFactor": 2,
1663           "legendFormat": "{{instance}}",
1664           "metric": "process_cpu_secondspersec",
1665           "refId": "A",
1666           "step": 4
1667         }
1668       ],
1669       "title": "CPU Usage",
1670       "type": "timeseries"
1671     },
1672     {
1673       "datasource": {
1674         "type": "prometheus",
1675         "uid": "PBFA97CFB590B2093"
1676       },
1677       "fieldConfig": {
1678         "defaults": {
1679           "color": {
1680             "mode": "palette-classic"
1681           },
1682           "custom": {
1683             "axisCenteredZero": false,
1684             "axisColorMode": "text",
1685             "axisLabel": "Memory",
1686             "axisPlacement": "auto",
1687             "barAlignment": 0,
1688             "drawStyle": "line",
1689             "fillOpacity": 10,
1690             "gradientMode": "none",
1691             "hideFrom": {
1692               "legend": false,
1693               "tooltip": false,
1694               "viz": false
1695             },
1696             "lineInterpolation": "linear",
1697             "lineWidth": 2,
1698             "pointSize": 5,
1699             "scaleDistribution": {
1700               "type": "linear"
1701             },
1702             "showPoints": "never",
1703             "spanNulls": false,
1704             "stacking": {
1705               "group": "A",
1706               "mode": "none"
1707             },
1708             "thresholdsStyle": {
1709               "mode": "off"
1710             }
1711           },
1712           "links": [],
1713           "mappings": [],
1714           "thresholds": {
1715             "mode": "absolute",
1716             "steps": [
1717               {
1718                 "color": "green",
1719                 "value": null
1720               },
1721               {
1722                 "color": "red",
1723                 "value": 80
1724               }
1725             ]
1726           },
1727           "unit": "bytes"
1728         },
1729         "overrides": [
1730           {
1731             "matcher": {
1732               "id": "byName",
1733               "options": "localhost:7071"
1734             },
1735             "properties": [
1736               {
1737                 "id": "color",
1738                 "value": {
1739                   "fixedColor": "#BA43A9",
1740                   "mode": "fixed"
1741                 }
1742               }
1743             ]
1744           },
1745           {
1746             "__systemRef": "hideSeriesFrom",
1747             "matcher": {
1748               "id": "byNames",
1749               "options": {
1750                 "mode": "exclude",
1751                 "names": [
1752                   "kafka-2:80"
1753                 ],
1754                 "prefix": "All except:",
1755                 "readOnly": true
1756               }
1757             },
1758             "properties": [
1759               {
1760                 "id": "custom.hideFrom",
1761                 "value": {
1762                   "legend": false,
1763                   "tooltip": false,
1764                   "viz": true
1765                 }
1766               }
1767             ]
1768           }
1769         ]
1770       },
1771       "gridPos": {
1772         "h": 12,
1773         "w": 8,
1774         "x": 8,
1775         "y": 27
1776       },
1777       "id": 2,
1778       "links": [],
1779       "options": {
1780         "legend": {
1781           "calcs": [
1782             "mean",
1783             "lastNotNull",
1784             "max"
1785           ],
1786           "displayMode": "table",
1787           "placement": "bottom",
1788           "showLegend": true
1789         },
1790         "tooltip": {
1791           "mode": "single",
1792           "sort": "none"
1793         }
1794       },
1795       "pluginVersion": "8.1.3",
1796       "targets": [
1797         {
1798           "datasource": {
1799             "type": "prometheus",
1800             "uid": "PBFA97CFB590B2093"
1801           },
1802           "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1803           "interval": "",
1804           "intervalFactor": 2,
1805           "legendFormat": "{{instance}}",
1806           "metric": "jvm_memory_bytes_used",
1807           "refId": "A",
1808           "step": 4
1809         },
1810         {
1811           "datasource": {
1812             "type": "prometheus",
1813             "uid": "PBFA97CFB590B2093"
1814           },
1815           "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1816           "interval": "",
1817           "legendFormat": "{{instance}}",
1818           "refId": "B"
1819         }
1820       ],
1821       "title": "JVM Memory Used",
1822       "type": "timeseries"
1823     },
1824     {
1825       "datasource": {
1826         "type": "prometheus",
1827         "uid": "PBFA97CFB590B2093"
1828       },
1829       "fieldConfig": {
1830         "defaults": {
1831           "color": {
1832             "mode": "palette-classic"
1833           },
1834           "custom": {
1835             "axisCenteredZero": false,
1836             "axisColorMode": "text",
1837             "axisLabel": "% time in GC",
1838             "axisPlacement": "auto",
1839             "barAlignment": 0,
1840             "drawStyle": "line",
1841             "fillOpacity": 10,
1842             "gradientMode": "none",
1843             "hideFrom": {
1844               "legend": false,
1845               "tooltip": false,
1846               "viz": false
1847             },
1848             "lineInterpolation": "linear",
1849             "lineWidth": 2,
1850             "pointSize": 5,
1851             "scaleDistribution": {
1852               "type": "linear"
1853             },
1854             "showPoints": "never",
1855             "spanNulls": false,
1856             "stacking": {
1857               "group": "A",
1858               "mode": "none"
1859             },
1860             "thresholdsStyle": {
1861               "mode": "off"
1862             }
1863           },
1864           "links": [],
1865           "mappings": [],
1866           "thresholds": {
1867             "mode": "absolute",
1868             "steps": [
1869               {
1870                 "color": "green",
1871                 "value": null
1872               },
1873               {
1874                 "color": "red",
1875                 "value": 80
1876               }
1877             ]
1878           },
1879           "unit": "percentunit"
1880         },
1881         "overrides": [
1882           {
1883             "matcher": {
1884               "id": "byName",
1885               "options": "localhost:7071"
1886             },
1887             "properties": [
1888               {
1889                 "id": "color",
1890                 "value": {
1891                   "fixedColor": "#890F02",
1892                   "mode": "fixed"
1893                 }
1894               }
1895             ]
1896           }
1897         ]
1898       },
1899       "gridPos": {
1900         "h": 12,
1901         "w": 8,
1902         "x": 16,
1903         "y": 27
1904       },
1905       "id": 3,
1906       "links": [],
1907       "options": {
1908         "legend": {
1909           "calcs": [
1910             "mean",
1911             "lastNotNull",
1912             "max"
1913           ],
1914           "displayMode": "table",
1915           "placement": "bottom",
1916           "showLegend": true
1917         },
1918         "tooltip": {
1919           "mode": "single",
1920           "sort": "none"
1921         }
1922       },
1923       "pluginVersion": "8.1.3",
1924       "targets": [
1925         {
1926           "datasource": {
1927             "type": "prometheus",
1928             "uid": "PBFA97CFB590B2093"
1929           },
1930           "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1931           "interval": "",
1932           "intervalFactor": 2,
1933           "legendFormat": "{{instance}}",
1934           "metric": "jvm_gc_collection_seconds_sum",
1935           "refId": "A",
1936           "step": 4
1937         }
1938       ],
1939       "title": "Time spent in GC",
1940       "type": "timeseries"
1941     },
1942     {
1943       "collapsed": true,
1944       "datasource": {
1945         "type": "datasource",
1946         "uid": "grafana"
1947       },
1948       "gridPos": {
1949         "h": 1,
1950         "w": 24,
1951         "x": 0,
1952         "y": 39
1953       },
1954       "id": 29,
1955       "panels": [
1956         {
1957           "datasource": {
1958             "type": "prometheus",
1959             "uid": "PBFA97CFB590B2093"
1960           },
1961           "fieldConfig": {
1962             "defaults": {
1963               "color": {
1964                 "mode": "palette-classic"
1965               },
1966               "custom": {
1967                 "axisLabel": "Messages/s",
1968                 "axisPlacement": "auto",
1969                 "barAlignment": 0,
1970                 "drawStyle": "line",
1971                 "fillOpacity": 10,
1972                 "gradientMode": "none",
1973                 "hideFrom": {
1974                   "legend": false,
1975                   "tooltip": false,
1976                   "viz": false
1977                 },
1978                 "lineInterpolation": "linear",
1979                 "lineWidth": 2,
1980                 "pointSize": 5,
1981                 "scaleDistribution": {
1982                   "type": "linear"
1983                 },
1984                 "showPoints": "never",
1985                 "spanNulls": false,
1986                 "stacking": {
1987                   "group": "A",
1988                   "mode": "none"
1989                 },
1990                 "thresholdsStyle": {
1991                   "mode": "off"
1992                 }
1993               },
1994               "links": [],
1995               "mappings": [],
1996               "min": 0,
1997               "thresholds": {
1998                 "mode": "absolute",
1999                 "steps": [
2000                   {
2001                     "color": "green"
2002                   }
2003                 ]
2004               },
2005               "unit": "cps"
2006             },
2007             "overrides": []
2008           },
2009           "gridPos": {
2010             "h": 8,
2011             "w": 8,
2012             "x": 0,
2013             "y": 24
2014           },
2015           "id": 4,
2016           "links": [],
2017           "options": {
2018             "legend": {
2019               "calcs": [
2020                 "mean",
2021                 "lastNotNull",
2022                 "max"
2023               ],
2024               "displayMode": "table",
2025               "placement": "bottom",
2026               "showLegend": true
2027             },
2028             "tooltip": {
2029               "mode": "single"
2030             }
2031           },
2032           "pluginVersion": "8.1.3",
2033           "targets": [
2034             {
2035               "datasource": {
2036                 "type": "prometheus",
2037                 "uid": "PBFA97CFB590B2093"
2038               },
2039               "exemplar": true,
2040               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2041               "interval": "",
2042               "intervalFactor": 2,
2043               "legendFormat": "bytes/sec",
2044               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2045               "refId": "A",
2046               "step": 4
2047             }
2048           ],
2049           "title": "Messages In",
2050           "type": "timeseries"
2051         },
2052         {
2053           "datasource": {
2054             "type": "prometheus",
2055             "uid": "PBFA97CFB590B2093"
2056           },
2057           "fieldConfig": {
2058             "defaults": {
2059               "color": {
2060                 "mode": "palette-classic"
2061               },
2062               "custom": {
2063                 "axisLabel": "Bytes/s",
2064                 "axisPlacement": "auto",
2065                 "barAlignment": 0,
2066                 "drawStyle": "line",
2067                 "fillOpacity": 10,
2068                 "gradientMode": "none",
2069                 "hideFrom": {
2070                   "legend": false,
2071                   "tooltip": false,
2072                   "viz": false
2073                 },
2074                 "lineInterpolation": "linear",
2075                 "lineWidth": 2,
2076                 "pointSize": 5,
2077                 "scaleDistribution": {
2078                   "type": "linear"
2079                 },
2080                 "showPoints": "never",
2081                 "spanNulls": false,
2082                 "stacking": {
2083                   "group": "A",
2084                   "mode": "none"
2085                 },
2086                 "thresholdsStyle": {
2087                   "mode": "off"
2088                 }
2089               },
2090               "links": [],
2091               "mappings": [],
2092               "min": 0,
2093               "thresholds": {
2094                 "mode": "absolute",
2095                 "steps": [
2096                   {
2097                     "color": "green"
2098                   },
2099                   {
2100                     "color": "red",
2101                     "value": 80
2102                   }
2103                 ]
2104               },
2105               "unit": "Bps"
2106             },
2107             "overrides": []
2108           },
2109           "gridPos": {
2110             "h": 8,
2111             "w": 8,
2112             "x": 8,
2113             "y": 24
2114           },
2115           "id": 5,
2116           "links": [],
2117           "options": {
2118             "legend": {
2119               "calcs": [
2120                 "mean",
2121                 "lastNotNull",
2122                 "max"
2123               ],
2124               "displayMode": "table",
2125               "placement": "bottom",
2126               "showLegend": true
2127             },
2128             "tooltip": {
2129               "mode": "single"
2130             }
2131           },
2132           "pluginVersion": "8.1.3",
2133           "targets": [
2134             {
2135               "datasource": {
2136                 "type": "prometheus",
2137                 "uid": "PBFA97CFB590B2093"
2138               },
2139               "exemplar": true,
2140               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2141               "format": "time_series",
2142               "interval": "",
2143               "intervalFactor": 2,
2144               "legendFormat": "bytes/sec",
2145               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2146               "refId": "A",
2147               "step": 4
2148             }
2149           ],
2150           "title": "Bytes In",
2151           "type": "timeseries"
2152         },
2153         {
2154           "datasource": {
2155             "type": "prometheus",
2156             "uid": "PBFA97CFB590B2093"
2157           },
2158           "fieldConfig": {
2159             "defaults": {
2160               "color": {
2161                 "mode": "palette-classic"
2162               },
2163               "custom": {
2164                 "axisLabel": "Bytes/s",
2165                 "axisPlacement": "auto",
2166                 "barAlignment": 0,
2167                 "drawStyle": "line",
2168                 "fillOpacity": 10,
2169                 "gradientMode": "none",
2170                 "hideFrom": {
2171                   "legend": false,
2172                   "tooltip": false,
2173                   "viz": false
2174                 },
2175                 "lineInterpolation": "linear",
2176                 "lineWidth": 2,
2177                 "pointSize": 5,
2178                 "scaleDistribution": {
2179                   "type": "linear"
2180                 },
2181                 "showPoints": "never",
2182                 "spanNulls": false,
2183                 "stacking": {
2184                   "group": "A",
2185                   "mode": "none"
2186                 },
2187                 "thresholdsStyle": {
2188                   "mode": "off"
2189                 }
2190               },
2191               "links": [],
2192               "mappings": [],
2193               "min": 0,
2194               "thresholds": {
2195                 "mode": "absolute",
2196                 "steps": [
2197                   {
2198                     "color": "green"
2199                   },
2200                   {
2201                     "color": "red",
2202                     "value": 80
2203                   }
2204                 ]
2205               },
2206               "unit": "Bps"
2207             },
2208             "overrides": []
2209           },
2210           "gridPos": {
2211             "h": 8,
2212             "w": 8,
2213             "x": 16,
2214             "y": 24
2215           },
2216           "id": 6,
2217           "links": [],
2218           "options": {
2219             "legend": {
2220               "calcs": [
2221                 "mean",
2222                 "lastNotNull",
2223                 "max"
2224               ],
2225               "displayMode": "table",
2226               "placement": "bottom",
2227               "showLegend": true
2228             },
2229             "tooltip": {
2230               "mode": "single"
2231             }
2232           },
2233           "pluginVersion": "8.1.3",
2234           "targets": [
2235             {
2236               "datasource": {
2237                 "type": "prometheus",
2238                 "uid": "PBFA97CFB590B2093"
2239               },
2240               "exemplar": true,
2241               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2242               "interval": "",
2243               "intervalFactor": 2,
2244               "legendFormat": "bytes/sec",
2245               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2246               "refId": "A",
2247               "step": 4
2248             }
2249           ],
2250           "title": "Bytes Out",
2251           "type": "timeseries"
2252         },
2253         {
2254           "datasource": {
2255             "type": "prometheus",
2256             "uid": "PBFA97CFB590B2093"
2257           },
2258           "fieldConfig": {
2259             "defaults": {
2260               "color": {
2261                 "mode": "palette-classic"
2262               },
2263               "custom": {
2264                 "axisLabel": "Messages/s",
2265                 "axisPlacement": "auto",
2266                 "barAlignment": 0,
2267                 "drawStyle": "line",
2268                 "fillOpacity": 10,
2269                 "gradientMode": "none",
2270                 "hideFrom": {
2271                   "legend": false,
2272                   "tooltip": false,
2273                   "viz": false
2274                 },
2275                 "lineInterpolation": "linear",
2276                 "lineWidth": 2,
2277                 "pointSize": 5,
2278                 "scaleDistribution": {
2279                   "type": "linear"
2280                 },
2281                 "showPoints": "never",
2282                 "spanNulls": false,
2283                 "stacking": {
2284                   "group": "A",
2285                   "mode": "none"
2286                 },
2287                 "thresholdsStyle": {
2288                   "mode": "off"
2289                 }
2290               },
2291               "links": [],
2292               "mappings": [],
2293               "thresholds": {
2294                 "mode": "absolute",
2295                 "steps": [
2296                   {
2297                     "color": "green"
2298                   },
2299                   {
2300                     "color": "red",
2301                     "value": 80
2302                   }
2303                 ]
2304               },
2305               "unit": "cps"
2306             },
2307             "overrides": []
2308           },
2309           "gridPos": {
2310             "h": 8,
2311             "w": 8,
2312             "x": 0,
2313             "y": 32
2314           },
2315           "id": 10,
2316           "links": [],
2317           "options": {
2318             "legend": {
2319               "calcs": [
2320                 "mean",
2321                 "lastNotNull",
2322                 "max"
2323               ],
2324               "displayMode": "table",
2325               "placement": "bottom",
2326               "showLegend": true
2327             },
2328             "tooltip": {
2329               "mode": "single"
2330             }
2331           },
2332           "pluginVersion": "8.1.3",
2333           "targets": [
2334             {
2335               "datasource": {
2336                 "type": "prometheus",
2337                 "uid": "PBFA97CFB590B2093"
2338               },
2339               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2340               "format": "time_series",
2341               "interval": "",
2342               "intervalFactor": 2,
2343               "legendFormat": "{{instance}}",
2344               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2345               "refId": "A",
2346               "step": 4
2347             }
2348           ],
2349           "title": "Messages In Per Broker",
2350           "type": "timeseries"
2351         },
2352         {
2353           "datasource": {
2354             "type": "prometheus",
2355             "uid": "PBFA97CFB590B2093"
2356           },
2357           "fieldConfig": {
2358             "defaults": {
2359               "color": {
2360                 "mode": "palette-classic"
2361               },
2362               "custom": {
2363                 "axisLabel": "Bytes/s",
2364                 "axisPlacement": "auto",
2365                 "barAlignment": 0,
2366                 "drawStyle": "line",
2367                 "fillOpacity": 10,
2368                 "gradientMode": "none",
2369                 "hideFrom": {
2370                   "legend": false,
2371                   "tooltip": false,
2372                   "viz": false
2373                 },
2374                 "lineInterpolation": "linear",
2375                 "lineWidth": 2,
2376                 "pointSize": 5,
2377                 "scaleDistribution": {
2378                   "type": "linear"
2379                 },
2380                 "showPoints": "never",
2381                 "spanNulls": false,
2382                 "stacking": {
2383                   "group": "A",
2384                   "mode": "none"
2385                 },
2386                 "thresholdsStyle": {
2387                   "mode": "off"
2388                 }
2389               },
2390               "links": [],
2391               "mappings": [],
2392               "thresholds": {
2393                 "mode": "absolute",
2394                 "steps": [
2395                   {
2396                     "color": "green"
2397                   },
2398                   {
2399                     "color": "red",
2400                     "value": 80
2401                   }
2402                 ]
2403               },
2404               "unit": "Bps"
2405             },
2406             "overrides": []
2407           },
2408           "gridPos": {
2409             "h": 8,
2410             "w": 8,
2411             "x": 8,
2412             "y": 32
2413           },
2414           "id": 7,
2415           "links": [],
2416           "options": {
2417             "legend": {
2418               "calcs": [
2419                 "mean",
2420                 "lastNotNull",
2421                 "max"
2422               ],
2423               "displayMode": "table",
2424               "placement": "bottom",
2425               "showLegend": true
2426             },
2427             "tooltip": {
2428               "mode": "single"
2429             }
2430           },
2431           "pluginVersion": "8.1.3",
2432           "targets": [
2433             {
2434               "datasource": {
2435                 "type": "prometheus",
2436                 "uid": "PBFA97CFB590B2093"
2437               },
2438               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2439               "format": "time_series",
2440               "interval": "",
2441               "intervalFactor": 2,
2442               "legendFormat": "{{instance}}",
2443               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2444               "refId": "A",
2445               "step": 4
2446             }
2447           ],
2448           "title": "Bytes In Per Broker",
2449           "type": "timeseries"
2450         },
2451         {
2452           "datasource": {
2453             "type": "prometheus",
2454             "uid": "PBFA97CFB590B2093"
2455           },
2456           "fieldConfig": {
2457             "defaults": {
2458               "color": {
2459                 "mode": "palette-classic"
2460               },
2461               "custom": {
2462                 "axisLabel": "",
2463                 "axisPlacement": "auto",
2464                 "barAlignment": 0,
2465                 "drawStyle": "line",
2466                 "fillOpacity": 10,
2467                 "gradientMode": "none",
2468                 "hideFrom": {
2469                   "legend": false,
2470                   "tooltip": false,
2471                   "viz": false
2472                 },
2473                 "lineInterpolation": "linear",
2474                 "lineWidth": 1,
2475                 "pointSize": 5,
2476                 "scaleDistribution": {
2477                   "type": "linear"
2478                 },
2479                 "showPoints": "never",
2480                 "spanNulls": true,
2481                 "stacking": {
2482                   "group": "A",
2483                   "mode": "none"
2484                 },
2485                 "thresholdsStyle": {
2486                   "mode": "off"
2487                 }
2488               },
2489               "links": [],
2490               "mappings": [],
2491               "min": 0,
2492               "thresholds": {
2493                 "mode": "absolute",
2494                 "steps": [
2495                   {
2496                     "color": "green"
2497                   },
2498                   {
2499                     "color": "red",
2500                     "value": 80
2501                   }
2502                 ]
2503               },
2504               "unit": "binBps"
2505             },
2506             "overrides": []
2507           },
2508           "gridPos": {
2509             "h": 8,
2510             "w": 8,
2511             "x": 16,
2512             "y": 32
2513           },
2514           "id": 9,
2515           "links": [],
2516           "options": {
2517             "legend": {
2518               "calcs": [
2519                 "mean",
2520                 "lastNotNull",
2521                 "max"
2522               ],
2523               "displayMode": "table",
2524               "placement": "bottom",
2525               "showLegend": true
2526             },
2527             "tooltip": {
2528               "mode": "single"
2529             }
2530           },
2531           "pluginVersion": "8.1.3",
2532           "targets": [
2533             {
2534               "datasource": {
2535                 "type": "prometheus",
2536                 "uid": "PBFA97CFB590B2093"
2537               },
2538               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2539               "format": "time_series",
2540               "interval": "",
2541               "intervalFactor": 1,
2542               "legendFormat": "{{instance}}",
2543               "refId": "A"
2544             }
2545           ],
2546           "title": "Bytes Out Per Broker",
2547           "type": "timeseries"
2548         }
2549       ],
2550       "targets": [
2551         {
2552           "datasource": {
2553             "type": "datasource",
2554             "uid": "grafana"
2555           },
2556           "refId": "A"
2557         }
2558       ],
2559       "title": "Throughput In/Out",
2560       "type": "row"
2561     },
2562     {
2563       "collapsed": false,
2564       "datasource": {
2565         "type": "datasource",
2566         "uid": "grafana"
2567       },
2568       "gridPos": {
2569         "h": 1,
2570         "w": 24,
2571         "x": 0,
2572         "y": 40
2573       },
2574       "id": 44,
2575       "panels": [],
2576       "targets": [
2577         {
2578           "datasource": {
2579             "type": "datasource",
2580             "uid": "grafana"
2581           },
2582           "refId": "A"
2583         }
2584       ],
2585       "title": "Thread utilization",
2586       "type": "row"
2587     },
2588     {
2589       "datasource": {
2590         "type": "prometheus",
2591         "uid": "PBFA97CFB590B2093"
2592       },
2593       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2594       "fieldConfig": {
2595         "defaults": {
2596           "color": {
2597             "mode": "palette-classic"
2598           },
2599           "custom": {
2600             "axisCenteredZero": false,
2601             "axisColorMode": "text",
2602             "axisLabel": "",
2603             "axisPlacement": "auto",
2604             "barAlignment": 0,
2605             "drawStyle": "line",
2606             "fillOpacity": 10,
2607             "gradientMode": "none",
2608             "hideFrom": {
2609               "legend": false,
2610               "tooltip": false,
2611               "viz": false
2612             },
2613             "lineInterpolation": "linear",
2614             "lineWidth": 1,
2615             "pointSize": 5,
2616             "scaleDistribution": {
2617               "type": "linear"
2618             },
2619             "showPoints": "never",
2620             "spanNulls": true,
2621             "stacking": {
2622               "group": "A",
2623               "mode": "none"
2624             },
2625             "thresholdsStyle": {
2626               "mode": "off"
2627             }
2628           },
2629           "links": [],
2630           "mappings": [],
2631           "thresholds": {
2632             "mode": "absolute",
2633             "steps": [
2634               {
2635                 "color": "green",
2636                 "value": null
2637               },
2638               {
2639                 "color": "red",
2640                 "value": 80
2641               }
2642             ]
2643           },
2644           "unit": "percentunit"
2645         },
2646         "overrides": []
2647       },
2648       "gridPos": {
2649         "h": 12,
2650         "w": 12,
2651         "x": 0,
2652         "y": 41
2653       },
2654       "id": 24,
2655       "options": {
2656         "legend": {
2657           "calcs": [
2658             "mean",
2659             "lastNotNull",
2660             "max"
2661           ],
2662           "displayMode": "table",
2663           "placement": "bottom",
2664           "showLegend": true
2665         },
2666         "tooltip": {
2667           "mode": "single",
2668           "sort": "none"
2669         }
2670       },
2671       "pluginVersion": "8.1.3",
2672       "targets": [
2673         {
2674           "datasource": {
2675             "type": "prometheus",
2676             "uid": "PBFA97CFB590B2093"
2677           },
2678           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2679           "interval": "",
2680           "legendFormat": "{{instance}}",
2681           "refId": "A"
2682         }
2683       ],
2684       "title": "Network Processor Avg Usage Percent",
2685       "type": "timeseries"
2686     },
2687     {
2688       "datasource": {
2689         "type": "prometheus",
2690         "uid": "PBFA97CFB590B2093"
2691       },
2692       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2693       "fieldConfig": {
2694         "defaults": {
2695           "color": {
2696             "mode": "palette-classic"
2697           },
2698           "custom": {
2699             "axisCenteredZero": false,
2700             "axisColorMode": "text",
2701             "axisLabel": "",
2702             "axisPlacement": "auto",
2703             "barAlignment": 0,
2704             "drawStyle": "line",
2705             "fillOpacity": 10,
2706             "gradientMode": "none",
2707             "hideFrom": {
2708               "legend": false,
2709               "tooltip": false,
2710               "viz": false
2711             },
2712             "lineInterpolation": "linear",
2713             "lineWidth": 1,
2714             "pointSize": 5,
2715             "scaleDistribution": {
2716               "type": "linear"
2717             },
2718             "showPoints": "never",
2719             "spanNulls": true,
2720             "stacking": {
2721               "group": "A",
2722               "mode": "none"
2723             },
2724             "thresholdsStyle": {
2725               "mode": "off"
2726             }
2727           },
2728           "links": [],
2729           "mappings": [],
2730           "min": 0,
2731           "thresholds": {
2732             "mode": "absolute",
2733             "steps": [
2734               {
2735                 "color": "green",
2736                 "value": null
2737               },
2738               {
2739                 "color": "red",
2740                 "value": 80
2741               }
2742             ]
2743           },
2744           "unit": "percentunit"
2745         },
2746         "overrides": []
2747       },
2748       "gridPos": {
2749         "h": 12,
2750         "w": 12,
2751         "x": 12,
2752         "y": 41
2753       },
2754       "id": 25,
2755       "options": {
2756         "legend": {
2757           "calcs": [
2758             "mean",
2759             "lastNotNull",
2760             "max"
2761           ],
2762           "displayMode": "table",
2763           "placement": "bottom",
2764           "showLegend": true
2765         },
2766         "tooltip": {
2767           "mode": "single",
2768           "sort": "none"
2769         }
2770       },
2771       "pluginVersion": "8.1.3",
2772       "targets": [
2773         {
2774           "datasource": {
2775             "type": "prometheus",
2776             "uid": "PBFA97CFB590B2093"
2777           },
2778           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2779           "interval": "",
2780           "legendFormat": "{{instance}}",
2781           "refId": "A"
2782         }
2783       ],
2784       "title": "Request Handler Avg Percent",
2785       "type": "timeseries"
2786     },
2787     {
2788       "collapsed": false,
2789       "datasource": {
2790         "type": "datasource",
2791         "uid": "grafana"
2792       },
2793       "gridPos": {
2794         "h": 1,
2795         "w": 24,
2796         "x": 0,
2797         "y": 53
2798       },
2799       "id": 82,
2800       "panels": [],
2801       "targets": [
2802         {
2803           "datasource": {
2804             "type": "datasource",
2805             "uid": "grafana"
2806           },
2807           "refId": "A"
2808         }
2809       ],
2810       "title": "Isr Shrinks / Expands",
2811       "type": "row"
2812     },
2813     {
2814       "datasource": {
2815         "type": "prometheus",
2816         "uid": "PBFA97CFB590B2093"
2817       },
2818       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2819       "fieldConfig": {
2820         "defaults": {
2821           "color": {
2822             "mode": "palette-classic"
2823           },
2824           "custom": {
2825             "axisCenteredZero": false,
2826             "axisColorMode": "text",
2827             "axisLabel": "",
2828             "axisPlacement": "auto",
2829             "barAlignment": 0,
2830             "drawStyle": "line",
2831             "fillOpacity": 10,
2832             "gradientMode": "none",
2833             "hideFrom": {
2834               "legend": false,
2835               "tooltip": false,
2836               "viz": false
2837             },
2838             "lineInterpolation": "linear",
2839             "lineWidth": 1,
2840             "pointSize": 5,
2841             "scaleDistribution": {
2842               "type": "linear"
2843             },
2844             "showPoints": "never",
2845             "spanNulls": true,
2846             "stacking": {
2847               "group": "A",
2848               "mode": "none"
2849             },
2850             "thresholdsStyle": {
2851               "mode": "off"
2852             }
2853           },
2854           "links": [],
2855           "mappings": [],
2856           "min": 0,
2857           "thresholds": {
2858             "mode": "absolute",
2859             "steps": [
2860               {
2861                 "color": "green",
2862                 "value": null
2863               },
2864               {
2865                 "color": "red",
2866                 "value": 80
2867               }
2868             ]
2869           },
2870           "unit": "ops"
2871         },
2872         "overrides": [
2873           {
2874             "__systemRef": "hideSeriesFrom",
2875             "matcher": {
2876               "id": "byNames",
2877               "options": {
2878                 "mode": "exclude",
2879                 "names": [
2880                   "kafka-2:80"
2881                 ],
2882                 "prefix": "All except:",
2883                 "readOnly": true
2884               }
2885             },
2886             "properties": [
2887               {
2888                 "id": "custom.hideFrom",
2889                 "value": {
2890                   "legend": false,
2891                   "tooltip": false,
2892                   "viz": true
2893                 }
2894               }
2895             ]
2896           }
2897         ]
2898       },
2899       "gridPos": {
2900         "h": 9,
2901         "w": 12,
2902         "x": 0,
2903         "y": 54
2904       },
2905       "id": 80,
2906       "options": {
2907         "legend": {
2908           "calcs": [
2909             "mean",
2910             "lastNotNull",
2911             "max"
2912           ],
2913           "displayMode": "table",
2914           "placement": "bottom",
2915           "showLegend": true
2916         },
2917         "tooltip": {
2918           "mode": "single",
2919           "sort": "none"
2920         }
2921       },
2922       "pluginVersion": "8.1.3",
2923       "targets": [
2924         {
2925           "datasource": {
2926             "type": "prometheus",
2927             "uid": "PBFA97CFB590B2093"
2928           },
2929           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2930           "interval": "",
2931           "legendFormat": "{{instance}}",
2932           "refId": "A"
2933         }
2934       ],
2935       "title": "IsrShrinks per Sec",
2936       "type": "timeseries"
2937     },
2938     {
2939       "datasource": {
2940         "type": "prometheus",
2941         "uid": "PBFA97CFB590B2093"
2942       },
2943       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2944       "fieldConfig": {
2945         "defaults": {
2946           "color": {
2947             "mode": "palette-classic"
2948           },
2949           "custom": {
2950             "axisCenteredZero": false,
2951             "axisColorMode": "text",
2952             "axisLabel": "",
2953             "axisPlacement": "auto",
2954             "barAlignment": 0,
2955             "drawStyle": "line",
2956             "fillOpacity": 10,
2957             "gradientMode": "none",
2958             "hideFrom": {
2959               "legend": false,
2960               "tooltip": false,
2961               "viz": false
2962             },
2963             "lineInterpolation": "linear",
2964             "lineWidth": 1,
2965             "pointSize": 5,
2966             "scaleDistribution": {
2967               "type": "linear"
2968             },
2969             "showPoints": "never",
2970             "spanNulls": true,
2971             "stacking": {
2972               "group": "A",
2973               "mode": "none"
2974             },
2975             "thresholdsStyle": {
2976               "mode": "off"
2977             }
2978           },
2979           "links": [],
2980           "mappings": [],
2981           "min": 0,
2982           "thresholds": {
2983             "mode": "absolute",
2984             "steps": [
2985               {
2986                 "color": "green",
2987                 "value": null
2988               },
2989               {
2990                 "color": "red",
2991                 "value": 80
2992               }
2993             ]
2994           },
2995           "unit": "ops"
2996         },
2997         "overrides": [
2998           {
2999             "__systemRef": "hideSeriesFrom",
3000             "matcher": {
3001               "id": "byNames",
3002               "options": {
3003                 "mode": "exclude",
3004                 "names": [
3005                   "kafka-2:80"
3006                 ],
3007                 "prefix": "All except:",
3008                 "readOnly": true
3009               }
3010             },
3011             "properties": [
3012               {
3013                 "id": "custom.hideFrom",
3014                 "value": {
3015                   "legend": false,
3016                   "tooltip": false,
3017                   "viz": true
3018                 }
3019               }
3020             ]
3021           }
3022         ]
3023       },
3024       "gridPos": {
3025         "h": 9,
3026         "w": 12,
3027         "x": 12,
3028         "y": 54
3029       },
3030       "id": 83,
3031       "options": {
3032         "legend": {
3033           "calcs": [
3034             "mean",
3035             "lastNotNull",
3036             "max"
3037           ],
3038           "displayMode": "table",
3039           "placement": "bottom",
3040           "showLegend": true
3041         },
3042         "tooltip": {
3043           "mode": "single",
3044           "sort": "none"
3045         }
3046       },
3047       "pluginVersion": "8.1.3",
3048       "targets": [
3049         {
3050           "datasource": {
3051             "type": "prometheus",
3052             "uid": "PBFA97CFB590B2093"
3053           },
3054           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3055           "hide": false,
3056           "interval": "",
3057           "legendFormat": "{{instance}}",
3058           "refId": "A"
3059         }
3060       ],
3061       "title": "IsrExpands per Sec",
3062       "type": "timeseries"
3063     },
3064     {
3065       "collapsed": true,
3066       "datasource": {
3067         "type": "datasource",
3068         "uid": "grafana"
3069       },
3070       "gridPos": {
3071         "h": 1,
3072         "w": 24,
3073         "x": 0,
3074         "y": 63
3075       },
3076       "id": 53,
3077       "panels": [
3078         {
3079           "datasource": {
3080             "type": "prometheus",
3081             "uid": "PBFA97CFB590B2093"
3082           },
3083           "fieldConfig": {
3084             "defaults": {
3085               "color": {
3086                 "mode": "palette-classic"
3087               },
3088               "custom": {
3089                 "axisLabel": "",
3090                 "axisPlacement": "auto",
3091                 "barAlignment": 0,
3092                 "drawStyle": "line",
3093                 "fillOpacity": 10,
3094                 "gradientMode": "none",
3095                 "hideFrom": {
3096                   "legend": false,
3097                   "tooltip": false,
3098                   "viz": false
3099                 },
3100                 "lineInterpolation": "linear",
3101                 "lineWidth": 1,
3102                 "pointSize": 5,
3103                 "scaleDistribution": {
3104                   "type": "linear"
3105                 },
3106                 "showPoints": "never",
3107                 "spanNulls": true,
3108                 "stacking": {
3109                   "group": "A",
3110                   "mode": "none"
3111                 },
3112                 "thresholdsStyle": {
3113                   "mode": "off"
3114                 }
3115               },
3116               "links": [],
3117               "mappings": [],
3118               "thresholds": {
3119                 "mode": "absolute",
3120                 "steps": [
3121                   {
3122                     "color": "green"
3123                   },
3124                   {
3125                     "color": "red",
3126                     "value": 80
3127                   }
3128                 ]
3129               },
3130               "unit": "bytes"
3131             },
3132             "overrides": []
3133           },
3134           "gridPos": {
3135             "h": 9,
3136             "w": 12,
3137             "x": 0,
3138             "y": 28
3139           },
3140           "id": 55,
3141           "options": {
3142             "legend": {
3143               "calcs": [
3144                 "mean",
3145                 "lastNotNull",
3146                 "max"
3147               ],
3148               "displayMode": "table",
3149               "placement": "bottom",
3150               "showLegend": true
3151             },
3152             "tooltip": {
3153               "mode": "single"
3154             }
3155           },
3156           "pluginVersion": "8.1.3",
3157           "targets": [
3158             {
3159               "datasource": {
3160                 "type": "prometheus",
3161                 "uid": "PBFA97CFB590B2093"
3162               },
3163               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3164               "legendFormat": "{{topic}}",
3165               "refId": "A"
3166             }
3167           ],
3168           "title": "Log size per Topic",
3169           "type": "timeseries"
3170         },
3171         {
3172           "datasource": {
3173             "type": "prometheus",
3174             "uid": "PBFA97CFB590B2093"
3175           },
3176           "fieldConfig": {
3177             "defaults": {
3178               "color": {
3179                 "mode": "palette-classic"
3180               },
3181               "custom": {
3182                 "axisLabel": "",
3183                 "axisPlacement": "auto",
3184                 "barAlignment": 0,
3185                 "drawStyle": "line",
3186                 "fillOpacity": 10,
3187                 "gradientMode": "none",
3188                 "hideFrom": {
3189                   "legend": false,
3190                   "tooltip": false,
3191                   "viz": false
3192                 },
3193                 "lineInterpolation": "linear",
3194                 "lineWidth": 1,
3195                 "pointSize": 5,
3196                 "scaleDistribution": {
3197                   "type": "linear"
3198                 },
3199                 "showPoints": "never",
3200                 "spanNulls": true,
3201                 "stacking": {
3202                   "group": "A",
3203                   "mode": "none"
3204                 },
3205                 "thresholdsStyle": {
3206                   "mode": "off"
3207                 }
3208               },
3209               "links": [],
3210               "mappings": [],
3211               "thresholds": {
3212                 "mode": "absolute",
3213                 "steps": [
3214                   {
3215                     "color": "green"
3216                   },
3217                   {
3218                     "color": "red",
3219                     "value": 80
3220                   }
3221                 ]
3222               },
3223               "unit": "bytes"
3224             },
3225             "overrides": []
3226           },
3227           "gridPos": {
3228             "h": 9,
3229             "w": 12,
3230             "x": 12,
3231             "y": 28
3232           },
3233           "id": 56,
3234           "options": {
3235             "legend": {
3236               "calcs": [
3237                 "mean",
3238                 "lastNotNull",
3239                 "max"
3240               ],
3241               "displayMode": "table",
3242               "placement": "bottom",
3243               "showLegend": true
3244             },
3245             "tooltip": {
3246               "mode": "single"
3247             }
3248           },
3249           "pluginVersion": "8.1.3",
3250           "targets": [
3251             {
3252               "datasource": {
3253                 "type": "prometheus",
3254                 "uid": "PBFA97CFB590B2093"
3255               },
3256               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3257               "legendFormat": "{{instance}}",
3258               "refId": "A"
3259             }
3260           ],
3261           "title": "Log size per Broker",
3262           "type": "timeseries"
3263         }
3264       ],
3265       "targets": [
3266         {
3267           "datasource": {
3268             "type": "datasource",
3269             "uid": "grafana"
3270           },
3271           "refId": "A"
3272         }
3273       ],
3274       "title": "Logs size",
3275       "type": "row"
3276     },
3277     {
3278       "collapsed": true,
3279       "datasource": {
3280         "type": "datasource",
3281         "uid": "grafana"
3282       },
3283       "gridPos": {
3284         "h": 1,
3285         "w": 24,
3286         "x": 0,
3287         "y": 64
3288       },
3289       "id": 58,
3290       "panels": [
3291         {
3292           "datasource": {
3293             "type": "prometheus",
3294             "uid": "PBFA97CFB590B2093"
3295           },
3296           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3297           "fieldConfig": {
3298             "defaults": {
3299               "color": {
3300                 "mode": "palette-classic"
3301               },
3302               "custom": {
3303                 "axisLabel": "",
3304                 "axisPlacement": "auto",
3305                 "barAlignment": 0,
3306                 "drawStyle": "line",
3307                 "fillOpacity": 10,
3308                 "gradientMode": "none",
3309                 "hideFrom": {
3310                   "legend": false,
3311                   "tooltip": false,
3312                   "viz": false
3313                 },
3314                 "lineInterpolation": "linear",
3315                 "lineWidth": 1,
3316                 "pointSize": 5,
3317                 "scaleDistribution": {
3318                   "type": "linear"
3319                 },
3320                 "showPoints": "never",
3321                 "spanNulls": true,
3322                 "stacking": {
3323                   "group": "A",
3324                   "mode": "none"
3325                 },
3326                 "thresholdsStyle": {
3327                   "mode": "off"
3328                 }
3329               },
3330               "links": [],
3331               "mappings": [],
3332               "thresholds": {
3333                 "mode": "absolute",
3334                 "steps": [
3335                   {
3336                     "color": "green"
3337                   },
3338                   {
3339                     "color": "red",
3340                     "value": 80
3341                   }
3342                 ]
3343               },
3344               "unit": "ms"
3345             },
3346             "overrides": []
3347           },
3348           "gridPos": {
3349             "h": 9,
3350             "w": 12,
3351             "x": 0,
3352             "y": 29
3353           },
3354           "id": 60,
3355           "options": {
3356             "legend": {
3357               "calcs": [
3358                 "mean",
3359                 "lastNotNull",
3360                 "max"
3361               ],
3362               "displayMode": "list",
3363               "placement": "bottom",
3364               "showLegend": true
3365             },
3366             "tooltip": {
3367               "mode": "single"
3368             }
3369           },
3370           "pluginVersion": "8.1.3",
3371           "targets": [
3372             {
3373               "datasource": {
3374                 "type": "prometheus",
3375                 "uid": "PBFA97CFB590B2093"
3376               },
3377               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3378               "hide": false,
3379               "legendFormat": "{{instance}} - {{quantile}}",
3380               "refId": "A"
3381             }
3382           ],
3383           "title": "Producer - RequestQueueTimeMs",
3384           "type": "timeseries"
3385         },
3386         {
3387           "datasource": {
3388             "type": "prometheus",
3389             "uid": "PBFA97CFB590B2093"
3390           },
3391           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3392           "fieldConfig": {
3393             "defaults": {
3394               "color": {
3395                 "mode": "palette-classic"
3396               },
3397               "custom": {
3398                 "axisLabel": "",
3399                 "axisPlacement": "auto",
3400                 "barAlignment": 0,
3401                 "drawStyle": "line",
3402                 "fillOpacity": 10,
3403                 "gradientMode": "none",
3404                 "hideFrom": {
3405                   "legend": false,
3406                   "tooltip": false,
3407                   "viz": false
3408                 },
3409                 "lineInterpolation": "linear",
3410                 "lineWidth": 1,
3411                 "pointSize": 5,
3412                 "scaleDistribution": {
3413                   "type": "linear"
3414                 },
3415                 "showPoints": "never",
3416                 "spanNulls": true,
3417                 "stacking": {
3418                   "group": "A",
3419                   "mode": "none"
3420                 },
3421                 "thresholdsStyle": {
3422                   "mode": "off"
3423                 }
3424               },
3425               "links": [],
3426               "mappings": [],
3427               "thresholds": {
3428                 "mode": "absolute",
3429                 "steps": [
3430                   {
3431                     "color": "green"
3432                   },
3433                   {
3434                     "color": "red",
3435                     "value": 80
3436                   }
3437                 ]
3438               },
3439               "unit": "ms"
3440             },
3441             "overrides": []
3442           },
3443           "gridPos": {
3444             "h": 9,
3445             "w": 12,
3446             "x": 12,
3447             "y": 29
3448           },
3449           "id": 61,
3450           "options": {
3451             "legend": {
3452               "calcs": [
3453                 "mean",
3454                 "lastNotNull",
3455                 "max"
3456               ],
3457               "displayMode": "list",
3458               "placement": "bottom",
3459               "showLegend": true
3460             },
3461             "tooltip": {
3462               "mode": "single"
3463             }
3464           },
3465           "pluginVersion": "8.1.3",
3466           "targets": [
3467             {
3468               "datasource": {
3469                 "type": "prometheus",
3470                 "uid": "PBFA97CFB590B2093"
3471               },
3472               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3473               "hide": false,
3474               "legendFormat": "{{instance}} - {{quantile}}",
3475               "refId": "A"
3476             }
3477           ],
3478           "title": "Producer - LocalTimeMs",
3479           "type": "timeseries"
3480         },
3481         {
3482           "datasource": {
3483             "type": "prometheus",
3484             "uid": "PBFA97CFB590B2093"
3485           },
3486           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3487           "fieldConfig": {
3488             "defaults": {
3489               "color": {
3490                 "mode": "palette-classic"
3491               },
3492               "custom": {
3493                 "axisLabel": "",
3494                 "axisPlacement": "auto",
3495                 "barAlignment": 0,
3496                 "drawStyle": "line",
3497                 "fillOpacity": 10,
3498                 "gradientMode": "none",
3499                 "hideFrom": {
3500                   "legend": false,
3501                   "tooltip": false,
3502                   "viz": false
3503                 },
3504                 "lineInterpolation": "linear",
3505                 "lineWidth": 1,
3506                 "pointSize": 5,
3507                 "scaleDistribution": {
3508                   "type": "linear"
3509                 },
3510                 "showPoints": "never",
3511                 "spanNulls": true,
3512                 "stacking": {
3513                   "group": "A",
3514                   "mode": "none"
3515                 },
3516                 "thresholdsStyle": {
3517                   "mode": "off"
3518                 }
3519               },
3520               "links": [],
3521               "mappings": [],
3522               "thresholds": {
3523                 "mode": "absolute",
3524                 "steps": [
3525                   {
3526                     "color": "green"
3527                   },
3528                   {
3529                     "color": "red",
3530                     "value": 80
3531                   }
3532                 ]
3533               },
3534               "unit": "ms"
3535             },
3536             "overrides": []
3537           },
3538           "gridPos": {
3539             "h": 10,
3540             "w": 8,
3541             "x": 0,
3542             "y": 38
3543           },
3544           "id": 62,
3545           "options": {
3546             "legend": {
3547               "calcs": [
3548                 "mean",
3549                 "lastNotNull",
3550                 "max"
3551               ],
3552               "displayMode": "table",
3553               "placement": "bottom",
3554               "showLegend": true
3555             },
3556             "tooltip": {
3557               "mode": "single"
3558             }
3559           },
3560           "pluginVersion": "8.1.3",
3561           "targets": [
3562             {
3563               "datasource": {
3564                 "type": "prometheus",
3565                 "uid": "PBFA97CFB590B2093"
3566               },
3567               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3568               "hide": false,
3569               "legendFormat": "{{instance}} - {{quantile}}",
3570               "refId": "A"
3571             }
3572           ],
3573           "title": "Producer - RemoteTimeMs",
3574           "type": "timeseries"
3575         },
3576         {
3577           "datasource": {
3578             "type": "prometheus",
3579             "uid": "PBFA97CFB590B2093"
3580           },
3581           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3582           "fieldConfig": {
3583             "defaults": {
3584               "color": {
3585                 "mode": "palette-classic"
3586               },
3587               "custom": {
3588                 "axisLabel": "",
3589                 "axisPlacement": "auto",
3590                 "barAlignment": 0,
3591                 "drawStyle": "line",
3592                 "fillOpacity": 10,
3593                 "gradientMode": "none",
3594                 "hideFrom": {
3595                   "legend": false,
3596                   "tooltip": false,
3597                   "viz": false
3598                 },
3599                 "lineInterpolation": "linear",
3600                 "lineWidth": 1,
3601                 "pointSize": 5,
3602                 "scaleDistribution": {
3603                   "type": "linear"
3604                 },
3605                 "showPoints": "never",
3606                 "spanNulls": true,
3607                 "stacking": {
3608                   "group": "A",
3609                   "mode": "none"
3610                 },
3611                 "thresholdsStyle": {
3612                   "mode": "off"
3613                 }
3614               },
3615               "links": [],
3616               "mappings": [],
3617               "thresholds": {
3618                 "mode": "absolute",
3619                 "steps": [
3620                   {
3621                     "color": "green"
3622                   },
3623                   {
3624                     "color": "red",
3625                     "value": 80
3626                   }
3627                 ]
3628               },
3629               "unit": "ms"
3630             },
3631             "overrides": []
3632           },
3633           "gridPos": {
3634             "h": 10,
3635             "w": 8,
3636             "x": 8,
3637             "y": 38
3638           },
3639           "id": 63,
3640           "options": {
3641             "legend": {
3642               "calcs": [
3643                 "mean",
3644                 "lastNotNull",
3645                 "max"
3646               ],
3647               "displayMode": "table",
3648               "placement": "bottom",
3649               "showLegend": true
3650             },
3651             "tooltip": {
3652               "mode": "single"
3653             }
3654           },
3655           "pluginVersion": "8.1.3",
3656           "targets": [
3657             {
3658               "datasource": {
3659                 "type": "prometheus",
3660                 "uid": "PBFA97CFB590B2093"
3661               },
3662               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3663               "hide": false,
3664               "legendFormat": "{{instance}} - {{quantile}}",
3665               "refId": "A"
3666             }
3667           ],
3668           "title": "Producer - ResponseQueueTimeMs",
3669           "type": "timeseries"
3670         },
3671         {
3672           "datasource": {
3673             "type": "prometheus",
3674             "uid": "PBFA97CFB590B2093"
3675           },
3676           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3677           "fieldConfig": {
3678             "defaults": {
3679               "color": {
3680                 "mode": "palette-classic"
3681               },
3682               "custom": {
3683                 "axisLabel": "",
3684                 "axisPlacement": "auto",
3685                 "barAlignment": 0,
3686                 "drawStyle": "line",
3687                 "fillOpacity": 10,
3688                 "gradientMode": "none",
3689                 "hideFrom": {
3690                   "legend": false,
3691                   "tooltip": false,
3692                   "viz": false
3693                 },
3694                 "lineInterpolation": "linear",
3695                 "lineWidth": 1,
3696                 "pointSize": 5,
3697                 "scaleDistribution": {
3698                   "type": "linear"
3699                 },
3700                 "showPoints": "never",
3701                 "spanNulls": true,
3702                 "stacking": {
3703                   "group": "A",
3704                   "mode": "none"
3705                 },
3706                 "thresholdsStyle": {
3707                   "mode": "off"
3708                 }
3709               },
3710               "links": [],
3711               "mappings": [],
3712               "min": 0,
3713               "thresholds": {
3714                 "mode": "absolute",
3715                 "steps": [
3716                   {
3717                     "color": "green"
3718                   },
3719                   {
3720                     "color": "red",
3721                     "value": 80
3722                   }
3723                 ]
3724               },
3725               "unit": "ms"
3726             },
3727             "overrides": []
3728           },
3729           "gridPos": {
3730             "h": 10,
3731             "w": 8,
3732             "x": 16,
3733             "y": 38
3734           },
3735           "id": 64,
3736           "options": {
3737             "legend": {
3738               "calcs": [
3739                 "mean",
3740                 "lastNotNull",
3741                 "max"
3742               ],
3743               "displayMode": "table",
3744               "placement": "bottom",
3745               "showLegend": true
3746             },
3747             "tooltip": {
3748               "mode": "single"
3749             }
3750           },
3751           "pluginVersion": "8.1.3",
3752           "targets": [
3753             {
3754               "datasource": {
3755                 "type": "prometheus",
3756                 "uid": "PBFA97CFB590B2093"
3757               },
3758               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3759               "hide": false,
3760               "legendFormat": "{{instance}} - {{quantile}}",
3761               "refId": "A"
3762             }
3763           ],
3764           "title": "Producer - ResponseSendTimeMs",
3765           "type": "timeseries"
3766         }
3767       ],
3768       "targets": [
3769         {
3770           "datasource": {
3771             "type": "datasource",
3772             "uid": "grafana"
3773           },
3774           "refId": "A"
3775         }
3776       ],
3777       "title": "Producer Performance",
3778       "type": "row"
3779     },
3780     {
3781       "collapsed": true,
3782       "datasource": {
3783         "type": "datasource",
3784         "uid": "grafana"
3785       },
3786       "gridPos": {
3787         "h": 1,
3788         "w": 24,
3789         "x": 0,
3790         "y": 65
3791       },
3792       "id": 68,
3793       "panels": [
3794         {
3795           "datasource": {
3796             "type": "prometheus",
3797             "uid": "PBFA97CFB590B2093"
3798           },
3799           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3800           "fieldConfig": {
3801             "defaults": {
3802               "color": {
3803                 "mode": "palette-classic"
3804               },
3805               "custom": {
3806                 "axisLabel": "",
3807                 "axisPlacement": "auto",
3808                 "barAlignment": 0,
3809                 "drawStyle": "line",
3810                 "fillOpacity": 10,
3811                 "gradientMode": "none",
3812                 "hideFrom": {
3813                   "legend": false,
3814                   "tooltip": false,
3815                   "viz": false
3816                 },
3817                 "lineInterpolation": "linear",
3818                 "lineWidth": 1,
3819                 "pointSize": 5,
3820                 "scaleDistribution": {
3821                   "type": "linear"
3822                 },
3823                 "showPoints": "never",
3824                 "spanNulls": true,
3825                 "stacking": {
3826                   "group": "A",
3827                   "mode": "none"
3828                 },
3829                 "thresholdsStyle": {
3830                   "mode": "off"
3831                 }
3832               },
3833               "links": [],
3834               "mappings": [],
3835               "min": 0,
3836               "thresholds": {
3837                 "mode": "absolute",
3838                 "steps": [
3839                   {
3840                     "color": "green"
3841                   },
3842                   {
3843                     "color": "red",
3844                     "value": 80
3845                   }
3846                 ]
3847               },
3848               "unit": "ms"
3849             },
3850             "overrides": []
3851           },
3852           "gridPos": {
3853             "h": 9,
3854             "w": 12,
3855             "x": 0,
3856             "y": 30
3857           },
3858           "id": 69,
3859           "options": {
3860             "legend": {
3861               "calcs": [
3862                 "mean",
3863                 "lastNotNull",
3864                 "max"
3865               ],
3866               "displayMode": "list",
3867               "placement": "bottom",
3868               "showLegend": true
3869             },
3870             "tooltip": {
3871               "mode": "single"
3872             }
3873           },
3874           "pluginVersion": "8.1.3",
3875           "targets": [
3876             {
3877               "datasource": {
3878                 "type": "prometheus",
3879                 "uid": "PBFA97CFB590B2093"
3880               },
3881               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3882               "hide": false,
3883               "legendFormat": "{{instance}} - {{quantile}}",
3884               "refId": "A"
3885             }
3886           ],
3887           "title": "Consumer - RequestQueueTimeMs",
3888           "type": "timeseries"
3889         },
3890         {
3891           "datasource": {
3892             "type": "prometheus",
3893             "uid": "PBFA97CFB590B2093"
3894           },
3895           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3896           "fieldConfig": {
3897             "defaults": {
3898               "color": {
3899                 "mode": "palette-classic"
3900               },
3901               "custom": {
3902                 "axisLabel": "",
3903                 "axisPlacement": "auto",
3904                 "barAlignment": 0,
3905                 "drawStyle": "line",
3906                 "fillOpacity": 10,
3907                 "gradientMode": "none",
3908                 "hideFrom": {
3909                   "legend": false,
3910                   "tooltip": false,
3911                   "viz": false
3912                 },
3913                 "lineInterpolation": "linear",
3914                 "lineWidth": 1,
3915                 "pointSize": 5,
3916                 "scaleDistribution": {
3917                   "type": "linear"
3918                 },
3919                 "showPoints": "never",
3920                 "spanNulls": true,
3921                 "stacking": {
3922                   "group": "A",
3923                   "mode": "none"
3924                 },
3925                 "thresholdsStyle": {
3926                   "mode": "off"
3927                 }
3928               },
3929               "links": [],
3930               "mappings": [],
3931               "thresholds": {
3932                 "mode": "absolute",
3933                 "steps": [
3934                   {
3935                     "color": "green"
3936                   },
3937                   {
3938                     "color": "red",
3939                     "value": 80
3940                   }
3941                 ]
3942               },
3943               "unit": "ms"
3944             },
3945             "overrides": []
3946           },
3947           "gridPos": {
3948             "h": 9,
3949             "w": 12,
3950             "x": 12,
3951             "y": 30
3952           },
3953           "id": 70,
3954           "options": {
3955             "legend": {
3956               "calcs": [
3957                 "mean",
3958                 "lastNotNull",
3959                 "max"
3960               ],
3961               "displayMode": "list",
3962               "placement": "bottom",
3963               "showLegend": true
3964             },
3965             "tooltip": {
3966               "mode": "single"
3967             }
3968           },
3969           "pluginVersion": "8.1.3",
3970           "targets": [
3971             {
3972               "datasource": {
3973                 "type": "prometheus",
3974                 "uid": "PBFA97CFB590B2093"
3975               },
3976               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3977               "hide": false,
3978               "legendFormat": "{{instance}} - {{quantile}}",
3979               "refId": "A"
3980             }
3981           ],
3982           "title": "Consumer - LocalTimeMs",
3983           "type": "timeseries"
3984         },
3985         {
3986           "datasource": {
3987             "type": "prometheus",
3988             "uid": "PBFA97CFB590B2093"
3989           },
3990           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3991           "fieldConfig": {
3992             "defaults": {
3993               "color": {
3994                 "mode": "palette-classic"
3995               },
3996               "custom": {
3997                 "axisLabel": "",
3998                 "axisPlacement": "auto",
3999                 "barAlignment": 0,
4000                 "drawStyle": "line",
4001                 "fillOpacity": 10,
4002                 "gradientMode": "none",
4003                 "hideFrom": {
4004                   "legend": false,
4005                   "tooltip": false,
4006                   "viz": false
4007                 },
4008                 "lineInterpolation": "linear",
4009                 "lineWidth": 1,
4010                 "pointSize": 5,
4011                 "scaleDistribution": {
4012                   "type": "linear"
4013                 },
4014                 "showPoints": "never",
4015                 "spanNulls": true,
4016                 "stacking": {
4017                   "group": "A",
4018                   "mode": "none"
4019                 },
4020                 "thresholdsStyle": {
4021                   "mode": "off"
4022                 }
4023               },
4024               "links": [],
4025               "mappings": [],
4026               "thresholds": {
4027                 "mode": "absolute",
4028                 "steps": [
4029                   {
4030                     "color": "green"
4031                   },
4032                   {
4033                     "color": "red",
4034                     "value": 80
4035                   }
4036                 ]
4037               },
4038               "unit": "ms"
4039             },
4040             "overrides": []
4041           },
4042           "gridPos": {
4043             "h": 9,
4044             "w": 8,
4045             "x": 0,
4046             "y": 39
4047           },
4048           "id": 71,
4049           "options": {
4050             "legend": {
4051               "calcs": [
4052                 "mean",
4053                 "lastNotNull",
4054                 "max"
4055               ],
4056               "displayMode": "table",
4057               "placement": "bottom",
4058               "showLegend": true
4059             },
4060             "tooltip": {
4061               "mode": "single"
4062             }
4063           },
4064           "pluginVersion": "8.1.3",
4065           "targets": [
4066             {
4067               "datasource": {
4068                 "type": "prometheus",
4069                 "uid": "PBFA97CFB590B2093"
4070               },
4071               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4072               "hide": false,
4073               "legendFormat": "{{instance}} - {{quantile}}",
4074               "refId": "A"
4075             }
4076           ],
4077           "title": "Consumer - RemoteTimeMs",
4078           "type": "timeseries"
4079         },
4080         {
4081           "datasource": {
4082             "type": "prometheus",
4083             "uid": "PBFA97CFB590B2093"
4084           },
4085           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4086           "fieldConfig": {
4087             "defaults": {
4088               "color": {
4089                 "mode": "palette-classic"
4090               },
4091               "custom": {
4092                 "axisLabel": "",
4093                 "axisPlacement": "auto",
4094                 "barAlignment": 0,
4095                 "drawStyle": "line",
4096                 "fillOpacity": 10,
4097                 "gradientMode": "none",
4098                 "hideFrom": {
4099                   "legend": false,
4100                   "tooltip": false,
4101                   "viz": false
4102                 },
4103                 "lineInterpolation": "linear",
4104                 "lineWidth": 1,
4105                 "pointSize": 5,
4106                 "scaleDistribution": {
4107                   "type": "linear"
4108                 },
4109                 "showPoints": "never",
4110                 "spanNulls": true,
4111                 "stacking": {
4112                   "group": "A",
4113                   "mode": "none"
4114                 },
4115                 "thresholdsStyle": {
4116                   "mode": "off"
4117                 }
4118               },
4119               "links": [],
4120               "mappings": [],
4121               "min": 0,
4122               "thresholds": {
4123                 "mode": "absolute",
4124                 "steps": [
4125                   {
4126                     "color": "green"
4127                   },
4128                   {
4129                     "color": "red",
4130                     "value": 80
4131                   }
4132                 ]
4133               },
4134               "unit": "ms"
4135             },
4136             "overrides": []
4137           },
4138           "gridPos": {
4139             "h": 9,
4140             "w": 8,
4141             "x": 8,
4142             "y": 39
4143           },
4144           "id": 72,
4145           "options": {
4146             "legend": {
4147               "calcs": [
4148                 "mean",
4149                 "lastNotNull",
4150                 "max"
4151               ],
4152               "displayMode": "table",
4153               "placement": "bottom",
4154               "showLegend": true
4155             },
4156             "tooltip": {
4157               "mode": "single"
4158             }
4159           },
4160           "pluginVersion": "8.1.3",
4161           "targets": [
4162             {
4163               "datasource": {
4164                 "type": "prometheus",
4165                 "uid": "PBFA97CFB590B2093"
4166               },
4167               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4168               "hide": false,
4169               "legendFormat": "{{instance}} - {{quantile}}",
4170               "refId": "A"
4171             }
4172           ],
4173           "title": "Consumer - ResponseQueueTimeMs",
4174           "type": "timeseries"
4175         },
4176         {
4177           "datasource": {
4178             "type": "prometheus",
4179             "uid": "PBFA97CFB590B2093"
4180           },
4181           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4182           "fieldConfig": {
4183             "defaults": {
4184               "color": {
4185                 "mode": "palette-classic"
4186               },
4187               "custom": {
4188                 "axisLabel": "",
4189                 "axisPlacement": "auto",
4190                 "barAlignment": 0,
4191                 "drawStyle": "line",
4192                 "fillOpacity": 10,
4193                 "gradientMode": "none",
4194                 "hideFrom": {
4195                   "legend": false,
4196                   "tooltip": false,
4197                   "viz": false
4198                 },
4199                 "lineInterpolation": "linear",
4200                 "lineWidth": 1,
4201                 "pointSize": 5,
4202                 "scaleDistribution": {
4203                   "type": "linear"
4204                 },
4205                 "showPoints": "never",
4206                 "spanNulls": true,
4207                 "stacking": {
4208                   "group": "A",
4209                   "mode": "none"
4210                 },
4211                 "thresholdsStyle": {
4212                   "mode": "off"
4213                 }
4214               },
4215               "links": [],
4216               "mappings": [],
4217               "min": 0,
4218               "thresholds": {
4219                 "mode": "absolute",
4220                 "steps": [
4221                   {
4222                     "color": "green"
4223                   },
4224                   {
4225                     "color": "red",
4226                     "value": 80
4227                   }
4228                 ]
4229               },
4230               "unit": "ms"
4231             },
4232             "overrides": []
4233           },
4234           "gridPos": {
4235             "h": 9,
4236             "w": 8,
4237             "x": 16,
4238             "y": 39
4239           },
4240           "id": 73,
4241           "options": {
4242             "legend": {
4243               "calcs": [
4244                 "mean",
4245                 "lastNotNull",
4246                 "max"
4247               ],
4248               "displayMode": "table",
4249               "placement": "bottom",
4250               "showLegend": true
4251             },
4252             "tooltip": {
4253               "mode": "single"
4254             }
4255           },
4256           "pluginVersion": "8.1.3",
4257           "targets": [
4258             {
4259               "datasource": {
4260                 "type": "prometheus",
4261                 "uid": "PBFA97CFB590B2093"
4262               },
4263               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4264               "hide": false,
4265               "legendFormat": "{{instance}} - {{quantile}}",
4266               "refId": "A"
4267             }
4268           ],
4269           "title": "Consumer - ResponseSendTimeMs",
4270           "type": "timeseries"
4271         }
4272       ],
4273       "targets": [
4274         {
4275           "datasource": {
4276             "type": "datasource",
4277             "uid": "grafana"
4278           },
4279           "refId": "A"
4280         }
4281       ],
4282       "title": "Consumer Performance",
4283       "type": "row"
4284     },
4285     {
4286       "collapsed": true,
4287       "datasource": {
4288         "type": "datasource",
4289         "uid": "grafana"
4290       },
4291       "gridPos": {
4292         "h": 1,
4293         "w": 24,
4294         "x": 0,
4295         "y": 66
4296       },
4297       "id": 66,
4298       "panels": [
4299         {
4300           "datasource": {
4301             "type": "prometheus",
4302             "uid": "PBFA97CFB590B2093"
4303           },
4304           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4305           "fieldConfig": {
4306             "defaults": {
4307               "color": {
4308                 "mode": "palette-classic"
4309               },
4310               "custom": {
4311                 "axisLabel": "",
4312                 "axisPlacement": "auto",
4313                 "barAlignment": 0,
4314                 "drawStyle": "line",
4315                 "fillOpacity": 10,
4316                 "gradientMode": "none",
4317                 "hideFrom": {
4318                   "legend": false,
4319                   "tooltip": false,
4320                   "viz": false
4321                 },
4322                 "lineInterpolation": "linear",
4323                 "lineWidth": 1,
4324                 "pointSize": 5,
4325                 "scaleDistribution": {
4326                   "type": "linear"
4327                 },
4328                 "showPoints": "never",
4329                 "spanNulls": true,
4330                 "stacking": {
4331                   "group": "A",
4332                   "mode": "none"
4333                 },
4334                 "thresholdsStyle": {
4335                   "mode": "off"
4336                 }
4337               },
4338               "links": [],
4339               "mappings": [],
4340               "min": 0,
4341               "thresholds": {
4342                 "mode": "absolute",
4343                 "steps": [
4344                   {
4345                     "color": "green"
4346                   },
4347                   {
4348                     "color": "red",
4349                     "value": 80
4350                   }
4351                 ]
4352               },
4353               "unit": "ms"
4354             },
4355             "overrides": []
4356           },
4357           "gridPos": {
4358             "h": 9,
4359             "w": 12,
4360             "x": 0,
4361             "y": 31
4362           },
4363           "id": 74,
4364           "options": {
4365             "legend": {
4366               "calcs": [
4367                 "mean",
4368                 "lastNotNull",
4369                 "max"
4370               ],
4371               "displayMode": "list",
4372               "placement": "bottom",
4373               "showLegend": true
4374             },
4375             "tooltip": {
4376               "mode": "single"
4377             }
4378           },
4379           "pluginVersion": "8.1.3",
4380           "targets": [
4381             {
4382               "datasource": {
4383                 "type": "prometheus",
4384                 "uid": "PBFA97CFB590B2093"
4385               },
4386               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4387               "hide": false,
4388               "legendFormat": "{{instance}} - {{quantile}}",
4389               "refId": "A"
4390             }
4391           ],
4392           "title": "FetchFollower - RequestQueueTimeMs",
4393           "type": "timeseries"
4394         },
4395         {
4396           "datasource": {
4397             "type": "prometheus",
4398             "uid": "PBFA97CFB590B2093"
4399           },
4400           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4401           "fieldConfig": {
4402             "defaults": {
4403               "color": {
4404                 "mode": "palette-classic"
4405               },
4406               "custom": {
4407                 "axisLabel": "",
4408                 "axisPlacement": "auto",
4409                 "barAlignment": 0,
4410                 "drawStyle": "line",
4411                 "fillOpacity": 10,
4412                 "gradientMode": "none",
4413                 "hideFrom": {
4414                   "legend": false,
4415                   "tooltip": false,
4416                   "viz": false
4417                 },
4418                 "lineInterpolation": "linear",
4419                 "lineWidth": 1,
4420                 "pointSize": 5,
4421                 "scaleDistribution": {
4422                   "type": "linear"
4423                 },
4424                 "showPoints": "never",
4425                 "spanNulls": true,
4426                 "stacking": {
4427                   "group": "A",
4428                   "mode": "none"
4429                 },
4430                 "thresholdsStyle": {
4431                   "mode": "off"
4432                 }
4433               },
4434               "links": [],
4435               "mappings": [],
4436               "min": 0,
4437               "thresholds": {
4438                 "mode": "absolute",
4439                 "steps": [
4440                   {
4441                     "color": "green"
4442                   },
4443                   {
4444                     "color": "red",
4445                     "value": 80
4446                   }
4447                 ]
4448               },
4449               "unit": "ms"
4450             },
4451             "overrides": []
4452           },
4453           "gridPos": {
4454             "h": 9,
4455             "w": 12,
4456             "x": 12,
4457             "y": 31
4458           },
4459           "id": 75,
4460           "options": {
4461             "legend": {
4462               "calcs": [
4463                 "mean",
4464                 "lastNotNull",
4465                 "max"
4466               ],
4467               "displayMode": "list",
4468               "placement": "bottom",
4469               "showLegend": true
4470             },
4471             "tooltip": {
4472               "mode": "single"
4473             }
4474           },
4475           "pluginVersion": "8.1.3",
4476           "targets": [
4477             {
4478               "datasource": {
4479                 "type": "prometheus",
4480                 "uid": "PBFA97CFB590B2093"
4481               },
4482               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4483               "hide": false,
4484               "legendFormat": "{{instance}} - {{quantile}}",
4485               "refId": "A"
4486             }
4487           ],
4488           "title": "FetchFollower - LocalTimeMs",
4489           "type": "timeseries"
4490         },
4491         {
4492           "datasource": {
4493             "type": "prometheus",
4494             "uid": "PBFA97CFB590B2093"
4495           },
4496           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4497           "fieldConfig": {
4498             "defaults": {
4499               "color": {
4500                 "mode": "palette-classic"
4501               },
4502               "custom": {
4503                 "axisLabel": "",
4504                 "axisPlacement": "auto",
4505                 "barAlignment": 0,
4506                 "drawStyle": "line",
4507                 "fillOpacity": 10,
4508                 "gradientMode": "none",
4509                 "hideFrom": {
4510                   "legend": false,
4511                   "tooltip": false,
4512                   "viz": false
4513                 },
4514                 "lineInterpolation": "linear",
4515                 "lineWidth": 1,
4516                 "pointSize": 5,
4517                 "scaleDistribution": {
4518                   "type": "linear"
4519                 },
4520                 "showPoints": "never",
4521                 "spanNulls": true,
4522                 "stacking": {
4523                   "group": "A",
4524                   "mode": "none"
4525                 },
4526                 "thresholdsStyle": {
4527                   "mode": "off"
4528                 }
4529               },
4530               "links": [],
4531               "mappings": [],
4532               "thresholds": {
4533                 "mode": "absolute",
4534                 "steps": [
4535                   {
4536                     "color": "green"
4537                   },
4538                   {
4539                     "color": "red",
4540                     "value": 80
4541                   }
4542                 ]
4543               },
4544               "unit": "ms"
4545             },
4546             "overrides": []
4547           },
4548           "gridPos": {
4549             "h": 9,
4550             "w": 8,
4551             "x": 0,
4552             "y": 40
4553           },
4554           "id": 76,
4555           "options": {
4556             "legend": {
4557               "calcs": [
4558                 "mean",
4559                 "lastNotNull",
4560                 "max"
4561               ],
4562               "displayMode": "table",
4563               "placement": "bottom",
4564               "showLegend": true
4565             },
4566             "tooltip": {
4567               "mode": "single"
4568             }
4569           },
4570           "pluginVersion": "8.1.3",
4571           "targets": [
4572             {
4573               "datasource": {
4574                 "type": "prometheus",
4575                 "uid": "PBFA97CFB590B2093"
4576               },
4577               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4578               "hide": false,
4579               "legendFormat": "{{instance}} - {{quantile}}",
4580               "refId": "A"
4581             }
4582           ],
4583           "title": "FetchFollower - RemoteTimeMs",
4584           "type": "timeseries"
4585         },
4586         {
4587           "datasource": {
4588             "type": "prometheus",
4589             "uid": "PBFA97CFB590B2093"
4590           },
4591           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4592           "fieldConfig": {
4593             "defaults": {
4594               "color": {
4595                 "mode": "palette-classic"
4596               },
4597               "custom": {
4598                 "axisLabel": "",
4599                 "axisPlacement": "auto",
4600                 "barAlignment": 0,
4601                 "drawStyle": "line",
4602                 "fillOpacity": 10,
4603                 "gradientMode": "none",
4604                 "hideFrom": {
4605                   "legend": false,
4606                   "tooltip": false,
4607                   "viz": false
4608                 },
4609                 "lineInterpolation": "linear",
4610                 "lineWidth": 1,
4611                 "pointSize": 5,
4612                 "scaleDistribution": {
4613                   "type": "linear"
4614                 },
4615                 "showPoints": "never",
4616                 "spanNulls": true,
4617                 "stacking": {
4618                   "group": "A",
4619                   "mode": "none"
4620                 },
4621                 "thresholdsStyle": {
4622                   "mode": "off"
4623                 }
4624               },
4625               "links": [],
4626               "mappings": [],
4627               "thresholds": {
4628                 "mode": "absolute",
4629                 "steps": [
4630                   {
4631                     "color": "green"
4632                   },
4633                   {
4634                     "color": "red",
4635                     "value": 80
4636                   }
4637                 ]
4638               },
4639               "unit": "ms"
4640             },
4641             "overrides": []
4642           },
4643           "gridPos": {
4644             "h": 9,
4645             "w": 8,
4646             "x": 8,
4647             "y": 40
4648           },
4649           "id": 77,
4650           "options": {
4651             "legend": {
4652               "calcs": [
4653                 "mean",
4654                 "lastNotNull",
4655                 "max"
4656               ],
4657               "displayMode": "table",
4658               "placement": "bottom",
4659               "showLegend": true
4660             },
4661             "tooltip": {
4662               "mode": "single"
4663             }
4664           },
4665           "pluginVersion": "8.1.3",
4666           "targets": [
4667             {
4668               "datasource": {
4669                 "type": "prometheus",
4670                 "uid": "PBFA97CFB590B2093"
4671               },
4672               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4673               "hide": false,
4674               "legendFormat": "{{instance}} - {{quantile}}",
4675               "refId": "A"
4676             }
4677           ],
4678           "title": "FetchFollower - ResponseQueueTimeMs",
4679           "type": "timeseries"
4680         },
4681         {
4682           "datasource": {
4683             "type": "prometheus",
4684             "uid": "PBFA97CFB590B2093"
4685           },
4686           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4687           "fieldConfig": {
4688             "defaults": {
4689               "color": {
4690                 "mode": "palette-classic"
4691               },
4692               "custom": {
4693                 "axisLabel": "",
4694                 "axisPlacement": "auto",
4695                 "barAlignment": 0,
4696                 "drawStyle": "line",
4697                 "fillOpacity": 10,
4698                 "gradientMode": "none",
4699                 "hideFrom": {
4700                   "legend": false,
4701                   "tooltip": false,
4702                   "viz": false
4703                 },
4704                 "lineInterpolation": "linear",
4705                 "lineWidth": 1,
4706                 "pointSize": 5,
4707                 "scaleDistribution": {
4708                   "type": "linear"
4709                 },
4710                 "showPoints": "never",
4711                 "spanNulls": true,
4712                 "stacking": {
4713                   "group": "A",
4714                   "mode": "none"
4715                 },
4716                 "thresholdsStyle": {
4717                   "mode": "off"
4718                 }
4719               },
4720               "links": [],
4721               "mappings": [],
4722               "min": 0,
4723               "thresholds": {
4724                 "mode": "absolute",
4725                 "steps": [
4726                   {
4727                     "color": "green"
4728                   },
4729                   {
4730                     "color": "red",
4731                     "value": 80
4732                   }
4733                 ]
4734               },
4735               "unit": "ms"
4736             },
4737             "overrides": []
4738           },
4739           "gridPos": {
4740             "h": 9,
4741             "w": 8,
4742             "x": 16,
4743             "y": 40
4744           },
4745           "id": 78,
4746           "options": {
4747             "legend": {
4748               "calcs": [
4749                 "mean",
4750                 "lastNotNull",
4751                 "max"
4752               ],
4753               "displayMode": "table",
4754               "placement": "bottom",
4755               "showLegend": true
4756             },
4757             "tooltip": {
4758               "mode": "single"
4759             }
4760           },
4761           "pluginVersion": "8.1.3",
4762           "targets": [
4763             {
4764               "datasource": {
4765                 "type": "prometheus",
4766                 "uid": "PBFA97CFB590B2093"
4767               },
4768               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4769               "hide": false,
4770               "legendFormat": "{{instance}} - {{quantile}}",
4771               "refId": "A"
4772             }
4773           ],
4774           "title": "FetchFollower - ResponseSendTimeMs",
4775           "type": "timeseries"
4776         }
4777       ],
4778       "targets": [
4779         {
4780           "datasource": {
4781             "type": "datasource",
4782             "uid": "grafana"
4783           },
4784           "refId": "A"
4785         }
4786       ],
4787       "title": "Fetch Follower Performance",
4788       "type": "row"
4789     },
4790     {
4791       "collapsed": true,
4792       "datasource": {
4793         "type": "datasource",
4794         "uid": "grafana"
4795       },
4796       "gridPos": {
4797         "h": 1,
4798         "w": 24,
4799         "x": 0,
4800         "y": 67
4801       },
4802       "id": 102,
4803       "panels": [
4804         {
4805           "datasource": {
4806             "type": "prometheus",
4807             "uid": "PBFA97CFB590B2093"
4808           },
4809           "fieldConfig": {
4810             "defaults": {
4811               "color": {
4812                 "mode": "palette-classic"
4813               },
4814               "custom": {
4815                 "axisLabel": "",
4816                 "axisPlacement": "auto",
4817                 "barAlignment": 0,
4818                 "drawStyle": "line",
4819                 "fillOpacity": 10,
4820                 "gradientMode": "none",
4821                 "hideFrom": {
4822                   "legend": false,
4823                   "tooltip": false,
4824                   "viz": false
4825                 },
4826                 "lineInterpolation": "linear",
4827                 "lineWidth": 1,
4828                 "pointSize": 5,
4829                 "scaleDistribution": {
4830                   "type": "linear"
4831                 },
4832                 "showPoints": "never",
4833                 "spanNulls": true,
4834                 "stacking": {
4835                   "group": "A",
4836                   "mode": "none"
4837                 },
4838                 "thresholdsStyle": {
4839                   "mode": "off"
4840                 }
4841               },
4842               "links": [],
4843               "mappings": [],
4844               "thresholds": {
4845                 "mode": "absolute",
4846                 "steps": [
4847                   {
4848                     "color": "green"
4849                   },
4850                   {
4851                     "color": "red",
4852                     "value": 80
4853                   }
4854                 ]
4855               },
4856               "unit": "short"
4857             },
4858             "overrides": []
4859           },
4860           "gridPos": {
4861             "h": 8,
4862             "w": 12,
4863             "x": 0,
4864             "y": 32
4865           },
4866           "id": 98,
4867           "options": {
4868             "legend": {
4869               "calcs": [
4870                 "mean",
4871                 "lastNotNull",
4872                 "max"
4873               ],
4874               "displayMode": "table",
4875               "placement": "bottom",
4876               "showLegend": true
4877             },
4878             "tooltip": {
4879               "mode": "single"
4880             }
4881           },
4882           "pluginVersion": "8.1.3",
4883           "targets": [
4884             {
4885               "datasource": {
4886                 "type": "prometheus",
4887                 "uid": "PBFA97CFB590B2093"
4888               },
4889               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4890               "interval": "",
4891               "legendFormat": "{{listener}}",
4892               "refId": "A"
4893             }
4894           ],
4895           "title": "Connections count per listener",
4896           "type": "timeseries"
4897         },
4898         {
4899           "datasource": {
4900             "type": "prometheus",
4901             "uid": "PBFA97CFB590B2093"
4902           },
4903           "fieldConfig": {
4904             "defaults": {
4905               "color": {
4906                 "mode": "palette-classic"
4907               },
4908               "custom": {
4909                 "axisLabel": "",
4910                 "axisPlacement": "auto",
4911                 "barAlignment": 0,
4912                 "drawStyle": "line",
4913                 "fillOpacity": 10,
4914                 "gradientMode": "none",
4915                 "hideFrom": {
4916                   "legend": false,
4917                   "tooltip": false,
4918                   "viz": false
4919                 },
4920                 "lineInterpolation": "linear",
4921                 "lineWidth": 1,
4922                 "pointSize": 5,
4923                 "scaleDistribution": {
4924                   "type": "linear"
4925                 },
4926                 "showPoints": "never",
4927                 "spanNulls": true,
4928                 "stacking": {
4929                   "group": "A",
4930                   "mode": "none"
4931                 },
4932                 "thresholdsStyle": {
4933                   "mode": "off"
4934                 }
4935               },
4936               "links": [],
4937               "mappings": [],
4938               "thresholds": {
4939                 "mode": "absolute",
4940                 "steps": [
4941                   {
4942                     "color": "green"
4943                   },
4944                   {
4945                     "color": "red",
4946                     "value": 80
4947                   }
4948                 ]
4949               },
4950               "unit": "short"
4951             },
4952             "overrides": []
4953           },
4954           "gridPos": {
4955             "h": 8,
4956             "w": 12,
4957             "x": 12,
4958             "y": 32
4959           },
4960           "id": 100,
4961           "options": {
4962             "legend": {
4963               "calcs": [
4964                 "mean",
4965                 "lastNotNull",
4966                 "max"
4967               ],
4968               "displayMode": "table",
4969               "placement": "bottom",
4970               "showLegend": true
4971             },
4972             "tooltip": {
4973               "mode": "single"
4974             }
4975           },
4976           "pluginVersion": "8.1.3",
4977           "targets": [
4978             {
4979               "datasource": {
4980                 "type": "prometheus",
4981                 "uid": "PBFA97CFB590B2093"
4982               },
4983               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
4984               "interval": "",
4985               "legendFormat": "{{instance}}",
4986               "refId": "A"
4987             }
4988           ],
4989           "title": "Connections count per broker",
4990           "type": "timeseries"
4991         },
4992         {
4993           "datasource": {
4994             "type": "prometheus",
4995             "uid": "PBFA97CFB590B2093"
4996           },
4997           "fieldConfig": {
4998             "defaults": {
4999               "color": {
5000                 "mode": "palette-classic"
5001               },
5002               "custom": {
5003                 "axisLabel": "",
5004                 "axisPlacement": "auto",
5005                 "barAlignment": 0,
5006                 "drawStyle": "line",
5007                 "fillOpacity": 10,
5008                 "gradientMode": "none",
5009                 "hideFrom": {
5010                   "legend": false,
5011                   "tooltip": false,
5012                   "viz": false
5013                 },
5014                 "lineInterpolation": "linear",
5015                 "lineWidth": 1,
5016                 "pointSize": 5,
5017                 "scaleDistribution": {
5018                   "type": "linear"
5019                 },
5020                 "showPoints": "never",
5021                 "spanNulls": true,
5022                 "stacking": {
5023                   "group": "A",
5024                   "mode": "none"
5025                 },
5026                 "thresholdsStyle": {
5027                   "mode": "off"
5028                 }
5029               },
5030               "links": [],
5031               "mappings": [],
5032               "thresholds": {
5033                 "mode": "absolute",
5034                 "steps": [
5035                   {
5036                     "color": "green"
5037                   },
5038                   {
5039                     "color": "red",
5040                     "value": 80
5041                   }
5042                 ]
5043               },
5044               "unit": "short"
5045             },
5046             "overrides": []
5047           },
5048           "gridPos": {
5049             "h": 8,
5050             "w": 12,
5051             "x": 0,
5052             "y": 40
5053           },
5054           "id": 104,
5055           "options": {
5056             "legend": {
5057               "calcs": [
5058                 "mean",
5059                 "lastNotNull",
5060                 "max"
5061               ],
5062               "displayMode": "table",
5063               "placement": "bottom",
5064               "showLegend": true
5065             },
5066             "tooltip": {
5067               "mode": "single"
5068             }
5069           },
5070           "pluginVersion": "8.1.3",
5071           "targets": [
5072             {
5073               "datasource": {
5074                 "type": "prometheus",
5075                 "uid": "PBFA97CFB590B2093"
5076               },
5077               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5078               "interval": "",
5079               "legendFormat": "{{listener}}",
5080               "refId": "A"
5081             }
5082           ],
5083           "title": "Connections creation rate per listener",
5084           "type": "timeseries"
5085         },
5086         {
5087           "datasource": {
5088             "type": "prometheus",
5089             "uid": "PBFA97CFB590B2093"
5090           },
5091           "fieldConfig": {
5092             "defaults": {
5093               "color": {
5094                 "mode": "palette-classic"
5095               },
5096               "custom": {
5097                 "axisLabel": "",
5098                 "axisPlacement": "auto",
5099                 "barAlignment": 0,
5100                 "drawStyle": "line",
5101                 "fillOpacity": 10,
5102                 "gradientMode": "none",
5103                 "hideFrom": {
5104                   "legend": false,
5105                   "tooltip": false,
5106                   "viz": false
5107                 },
5108                 "lineInterpolation": "linear",
5109                 "lineWidth": 1,
5110                 "pointSize": 5,
5111                 "scaleDistribution": {
5112                   "type": "linear"
5113                 },
5114                 "showPoints": "never",
5115                 "spanNulls": true,
5116                 "stacking": {
5117                   "group": "A",
5118                   "mode": "none"
5119                 },
5120                 "thresholdsStyle": {
5121                   "mode": "off"
5122                 }
5123               },
5124               "links": [],
5125               "mappings": [],
5126               "thresholds": {
5127                 "mode": "absolute",
5128                 "steps": [
5129                   {
5130                     "color": "green"
5131                   },
5132                   {
5133                     "color": "red",
5134                     "value": 80
5135                   }
5136                 ]
5137               },
5138               "unit": "short"
5139             },
5140             "overrides": []
5141           },
5142           "gridPos": {
5143             "h": 8,
5144             "w": 12,
5145             "x": 12,
5146             "y": 40
5147           },
5148           "id": 106,
5149           "options": {
5150             "legend": {
5151               "calcs": [
5152                 "mean",
5153                 "lastNotNull",
5154                 "max"
5155               ],
5156               "displayMode": "table",
5157               "placement": "bottom",
5158               "showLegend": true
5159             },
5160             "tooltip": {
5161               "mode": "single"
5162             }
5163           },
5164           "pluginVersion": "8.1.3",
5165           "targets": [
5166             {
5167               "datasource": {
5168                 "type": "prometheus",
5169                 "uid": "PBFA97CFB590B2093"
5170               },
5171               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5172               "interval": "",
5173               "legendFormat": "{{instance}}",
5174               "refId": "A"
5175             }
5176           ],
5177           "title": "Connections creation rate per instance",
5178           "type": "timeseries"
5179         },
5180         {
5181           "datasource": {
5182             "type": "prometheus",
5183             "uid": "PBFA97CFB590B2093"
5184           },
5185           "fieldConfig": {
5186             "defaults": {
5187               "color": {
5188                 "mode": "palette-classic"
5189               },
5190               "custom": {
5191                 "axisLabel": "",
5192                 "axisPlacement": "auto",
5193                 "barAlignment": 0,
5194                 "drawStyle": "line",
5195                 "fillOpacity": 10,
5196                 "gradientMode": "none",
5197                 "hideFrom": {
5198                   "legend": false,
5199                   "tooltip": false,
5200                   "viz": false
5201                 },
5202                 "lineInterpolation": "linear",
5203                 "lineWidth": 1,
5204                 "pointSize": 5,
5205                 "scaleDistribution": {
5206                   "type": "linear"
5207                 },
5208                 "showPoints": "never",
5209                 "spanNulls": true,
5210                 "stacking": {
5211                   "group": "A",
5212                   "mode": "none"
5213                 },
5214                 "thresholdsStyle": {
5215                   "mode": "off"
5216                 }
5217               },
5218               "links": [],
5219               "mappings": [],
5220               "thresholds": {
5221                 "mode": "absolute",
5222                 "steps": [
5223                   {
5224                     "color": "green"
5225                   },
5226                   {
5227                     "color": "red",
5228                     "value": 80
5229                   }
5230                 ]
5231               },
5232               "unit": "short"
5233             },
5234             "overrides": []
5235           },
5236           "gridPos": {
5237             "h": 8,
5238             "w": 12,
5239             "x": 0,
5240             "y": 48
5241           },
5242           "id": 108,
5243           "options": {
5244             "legend": {
5245               "calcs": [
5246                 "mean",
5247                 "lastNotNull",
5248                 "max"
5249               ],
5250               "displayMode": "table",
5251               "placement": "bottom",
5252               "showLegend": true
5253             },
5254             "tooltip": {
5255               "mode": "single"
5256             }
5257           },
5258           "pluginVersion": "8.1.3",
5259           "targets": [
5260             {
5261               "datasource": {
5262                 "type": "prometheus",
5263                 "uid": "PBFA97CFB590B2093"
5264               },
5265               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5266               "interval": "",
5267               "legendFormat": "{{listener}}",
5268               "refId": "A"
5269             }
5270           ],
5271           "title": "Connections close rate per listener",
5272           "type": "timeseries"
5273         },
5274         {
5275           "datasource": {
5276             "type": "prometheus",
5277             "uid": "PBFA97CFB590B2093"
5278           },
5279           "fieldConfig": {
5280             "defaults": {
5281               "color": {
5282                 "mode": "palette-classic"
5283               },
5284               "custom": {
5285                 "axisLabel": "",
5286                 "axisPlacement": "auto",
5287                 "barAlignment": 0,
5288                 "drawStyle": "line",
5289                 "fillOpacity": 10,
5290                 "gradientMode": "none",
5291                 "hideFrom": {
5292                   "legend": false,
5293                   "tooltip": false,
5294                   "viz": false
5295                 },
5296                 "lineInterpolation": "linear",
5297                 "lineWidth": 1,
5298                 "pointSize": 5,
5299                 "scaleDistribution": {
5300                   "type": "linear"
5301                 },
5302                 "showPoints": "never",
5303                 "spanNulls": true,
5304                 "stacking": {
5305                   "group": "A",
5306                   "mode": "none"
5307                 },
5308                 "thresholdsStyle": {
5309                   "mode": "off"
5310                 }
5311               },
5312               "links": [],
5313               "mappings": [],
5314               "thresholds": {
5315                 "mode": "absolute",
5316                 "steps": [
5317                   {
5318                     "color": "green"
5319                   },
5320                   {
5321                     "color": "red",
5322                     "value": 80
5323                   }
5324                 ]
5325               },
5326               "unit": "short"
5327             },
5328             "overrides": []
5329           },
5330           "gridPos": {
5331             "h": 8,
5332             "w": 12,
5333             "x": 12,
5334             "y": 48
5335           },
5336           "id": 110,
5337           "options": {
5338             "legend": {
5339               "calcs": [
5340                 "mean",
5341                 "lastNotNull",
5342                 "max"
5343               ],
5344               "displayMode": "table",
5345               "placement": "bottom",
5346               "showLegend": true
5347             },
5348             "tooltip": {
5349               "mode": "single"
5350             }
5351           },
5352           "pluginVersion": "8.1.3",
5353           "targets": [
5354             {
5355               "datasource": {
5356                 "type": "prometheus",
5357                 "uid": "PBFA97CFB590B2093"
5358               },
5359               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5360               "interval": "",
5361               "legendFormat": "{{instance}}",
5362               "refId": "A"
5363             }
5364           ],
5365           "title": "Connections close rate per instance",
5366           "type": "timeseries"
5367         },
5368         {
5369           "datasource": {
5370             "type": "prometheus",
5371             "uid": "PBFA97CFB590B2093"
5372           },
5373           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5374           "fieldConfig": {
5375             "defaults": {
5376               "color": {
5377                 "mode": "palette-classic"
5378               },
5379               "custom": {
5380                 "axisLabel": "",
5381                 "axisPlacement": "auto",
5382                 "barAlignment": 0,
5383                 "drawStyle": "line",
5384                 "fillOpacity": 10,
5385                 "gradientMode": "none",
5386                 "hideFrom": {
5387                   "legend": false,
5388                   "tooltip": false,
5389                   "viz": false
5390                 },
5391                 "lineInterpolation": "linear",
5392                 "lineWidth": 1,
5393                 "pointSize": 5,
5394                 "scaleDistribution": {
5395                   "type": "linear"
5396                 },
5397                 "showPoints": "never",
5398                 "spanNulls": true,
5399                 "stacking": {
5400                   "group": "A",
5401                   "mode": "none"
5402                 },
5403                 "thresholdsStyle": {
5404                   "mode": "off"
5405                 }
5406               },
5407               "links": [],
5408               "mappings": [],
5409               "min": 0,
5410               "thresholds": {
5411                 "mode": "absolute",
5412                 "steps": [
5413                   {
5414                     "color": "green"
5415                   },
5416                   {
5417                     "color": "red",
5418                     "value": 80
5419                   }
5420                 ]
5421               },
5422               "unit": "percent"
5423             },
5424             "overrides": []
5425           },
5426           "gridPos": {
5427             "h": 8,
5428             "w": 12,
5429             "x": 0,
5430             "y": 56
5431           },
5432           "id": 112,
5433           "options": {
5434             "legend": {
5435               "calcs": [
5436                 "mean",
5437                 "lastNotNull",
5438                 "max"
5439               ],
5440               "displayMode": "table",
5441               "placement": "bottom",
5442               "showLegend": true
5443             },
5444             "tooltip": {
5445               "mode": "single"
5446             }
5447           },
5448           "pluginVersion": "8.1.3",
5449           "targets": [
5450             {
5451               "datasource": {
5452                 "type": "prometheus",
5453                 "uid": "PBFA97CFB590B2093"
5454               },
5455               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5456               "interval": "",
5457               "legendFormat": "{{instance}} - {{listener}}",
5458               "refId": "A"
5459             }
5460           ],
5461           "title": "Acceptor Blocked Percentage",
5462           "type": "timeseries"
5463         },
5464         {
5465           "datasource": {
5466             "type": "prometheus",
5467             "uid": "PBFA97CFB590B2093"
5468           },
5469           "fieldConfig": {
5470             "defaults": {
5471               "color": {
5472                 "mode": "palette-classic"
5473               },
5474               "custom": {
5475                 "axisLabel": "",
5476                 "axisPlacement": "auto",
5477                 "barAlignment": 0,
5478                 "drawStyle": "line",
5479                 "fillOpacity": 10,
5480                 "gradientMode": "none",
5481                 "hideFrom": {
5482                   "legend": false,
5483                   "tooltip": false,
5484                   "viz": false
5485                 },
5486                 "lineInterpolation": "linear",
5487                 "lineWidth": 1,
5488                 "pointSize": 5,
5489                 "scaleDistribution": {
5490                   "type": "linear"
5491                 },
5492                 "showPoints": "never",
5493                 "spanNulls": true,
5494                 "stacking": {
5495                   "group": "A",
5496                   "mode": "none"
5497                 },
5498                 "thresholdsStyle": {
5499                   "mode": "off"
5500                 }
5501               },
5502               "links": [],
5503               "mappings": [],
5504               "thresholds": {
5505                 "mode": "absolute",
5506                 "steps": [
5507                   {
5508                     "color": "green"
5509                   },
5510                   {
5511                     "color": "red",
5512                     "value": 80
5513                   }
5514                 ]
5515               },
5516               "unit": "short"
5517             },
5518             "overrides": []
5519           },
5520           "gridPos": {
5521             "h": 8,
5522             "w": 12,
5523             "x": 12,
5524             "y": 56
5525           },
5526           "id": 114,
5527           "options": {
5528             "legend": {
5529               "calcs": [
5530                 "mean",
5531                 "lastNotNull",
5532                 "max"
5533               ],
5534               "displayMode": "table",
5535               "placement": "bottom",
5536               "showLegend": true
5537             },
5538             "tooltip": {
5539               "mode": "single"
5540             }
5541           },
5542           "pluginVersion": "8.1.3",
5543           "targets": [
5544             {
5545               "datasource": {
5546                 "type": "prometheus",
5547                 "uid": "PBFA97CFB590B2093"
5548               },
5549               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5550               "interval": "",
5551               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5552               "refId": "A"
5553             }
5554           ],
5555           "title": "Connections per client version",
5556           "type": "timeseries"
5557         }
5558       ],
5559       "targets": [
5560         {
5561           "datasource": {
5562             "type": "datasource",
5563             "uid": "grafana"
5564           },
5565           "refId": "A"
5566         }
5567       ],
5568       "title": "Connections",
5569       "type": "row"
5570     },
5571     {
5572       "collapsed": true,
5573       "datasource": {
5574         "type": "datasource",
5575         "uid": "grafana"
5576       },
5577       "gridPos": {
5578         "h": 1,
5579         "w": 24,
5580         "x": 0,
5581         "y": 68
5582       },
5583       "id": 120,
5584       "panels": [
5585         {
5586           "datasource": {
5587             "type": "prometheus",
5588             "uid": "PBFA97CFB590B2093"
5589           },
5590           "description": "Number of consumer groups per group coordinator",
5591           "fieldConfig": {
5592             "defaults": {
5593               "color": {
5594                 "mode": "palette-classic"
5595               },
5596               "custom": {
5597                 "axisLabel": "",
5598                 "axisPlacement": "auto",
5599                 "barAlignment": 0,
5600                 "drawStyle": "line",
5601                 "fillOpacity": 10,
5602                 "gradientMode": "none",
5603                 "hideFrom": {
5604                   "legend": false,
5605                   "tooltip": false,
5606                   "viz": false
5607                 },
5608                 "lineInterpolation": "linear",
5609                 "lineWidth": 1,
5610                 "pointSize": 5,
5611                 "scaleDistribution": {
5612                   "type": "linear"
5613                 },
5614                 "showPoints": "never",
5615                 "spanNulls": true,
5616                 "stacking": {
5617                   "group": "A",
5618                   "mode": "none"
5619                 },
5620                 "thresholdsStyle": {
5621                   "mode": "off"
5622                 }
5623               },
5624               "links": [],
5625               "mappings": [],
5626               "thresholds": {
5627                 "mode": "absolute",
5628                 "steps": [
5629                   {
5630                     "color": "green"
5631                   },
5632                   {
5633                     "color": "red",
5634                     "value": 80
5635                   }
5636                 ]
5637               },
5638               "unit": "short"
5639             },
5640             "overrides": []
5641           },
5642           "gridPos": {
5643             "h": 8,
5644             "w": 12,
5645             "x": 0,
5646             "y": 33
5647           },
5648           "id": 116,
5649           "options": {
5650             "legend": {
5651               "calcs": [
5652                 "mean",
5653                 "lastNotNull",
5654                 "max"
5655               ],
5656               "displayMode": "table",
5657               "placement": "bottom",
5658               "showLegend": true
5659             },
5660             "tooltip": {
5661               "mode": "single"
5662             }
5663           },
5664           "pluginVersion": "8.1.3",
5665           "targets": [
5666             {
5667               "datasource": {
5668                 "type": "prometheus",
5669                 "uid": "PBFA97CFB590B2093"
5670               },
5671               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5672               "instant": false,
5673               "interval": "",
5674               "legendFormat": "{{instance}}",
5675               "refId": "A"
5676             }
5677           ],
5678           "title": "Consumer groups number per coordinator",
5679           "type": "timeseries"
5680         },
5681         {
5682           "datasource": {
5683             "type": "prometheus",
5684             "uid": "PBFA97CFB590B2093"
5685           },
5686           "description": "Number of consumer group per state",
5687           "fieldConfig": {
5688             "defaults": {
5689               "color": {
5690                 "mode": "palette-classic"
5691               },
5692               "custom": {
5693                 "axisLabel": "",
5694                 "axisPlacement": "auto",
5695                 "barAlignment": 0,
5696                 "drawStyle": "line",
5697                 "fillOpacity": 10,
5698                 "gradientMode": "none",
5699                 "hideFrom": {
5700                   "legend": false,
5701                   "tooltip": false,
5702                   "viz": false
5703                 },
5704                 "lineInterpolation": "linear",
5705                 "lineWidth": 1,
5706                 "pointSize": 5,
5707                 "scaleDistribution": {
5708                   "type": "linear"
5709                 },
5710                 "showPoints": "never",
5711                 "spanNulls": true,
5712                 "stacking": {
5713                   "group": "A",
5714                   "mode": "none"
5715                 },
5716                 "thresholdsStyle": {
5717                   "mode": "off"
5718                 }
5719               },
5720               "links": [],
5721               "mappings": [],
5722               "thresholds": {
5723                 "mode": "absolute",
5724                 "steps": [
5725                   {
5726                     "color": "green"
5727                   },
5728                   {
5729                     "color": "red",
5730                     "value": 80
5731                   }
5732                 ]
5733               },
5734               "unit": "short"
5735             },
5736             "overrides": []
5737           },
5738           "gridPos": {
5739             "h": 8,
5740             "w": 12,
5741             "x": 12,
5742             "y": 33
5743           },
5744           "id": 118,
5745           "options": {
5746             "legend": {
5747               "calcs": [
5748                 "mean",
5749                 "lastNotNull",
5750                 "max"
5751               ],
5752               "displayMode": "table",
5753               "placement": "bottom",
5754               "showLegend": true
5755             },
5756             "tooltip": {
5757               "mode": "single"
5758             }
5759           },
5760           "pluginVersion": "8.1.3",
5761           "targets": [
5762             {
5763               "datasource": {
5764                 "type": "prometheus",
5765                 "uid": "PBFA97CFB590B2093"
5766               },
5767               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5768               "instant": false,
5769               "interval": "",
5770               "legendFormat": "stable",
5771               "refId": "A"
5772             },
5773             {
5774               "datasource": {
5775                 "type": "prometheus",
5776                 "uid": "PBFA97CFB590B2093"
5777               },
5778               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5779               "interval": "",
5780               "legendFormat": "preparing-rebalance",
5781               "refId": "B"
5782             },
5783             {
5784               "datasource": {
5785                 "type": "prometheus",
5786                 "uid": "PBFA97CFB590B2093"
5787               },
5788               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5789               "interval": "",
5790               "legendFormat": "dead",
5791               "refId": "C"
5792             },
5793             {
5794               "datasource": {
5795                 "type": "prometheus",
5796                 "uid": "PBFA97CFB590B2093"
5797               },
5798               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5799               "interval": "",
5800               "legendFormat": "completing-rebalance",
5801               "refId": "D"
5802             },
5803             {
5804               "datasource": {
5805                 "type": "prometheus",
5806                 "uid": "PBFA97CFB590B2093"
5807               },
5808               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5809               "interval": "",
5810               "legendFormat": "empty",
5811               "refId": "E"
5812             }
5813           ],
5814           "title": "Nb consumer groups per state",
5815           "type": "timeseries"
5816         }
5817       ],
5818       "targets": [
5819         {
5820           "datasource": {
5821             "type": "datasource",
5822             "uid": "grafana"
5823           },
5824           "refId": "A"
5825         }
5826       ],
5827       "title": "Group Coordinator",
5828       "type": "row"
5829     },
5830     {
5831       "collapsed": true,
5832       "datasource": {
5833         "type": "datasource",
5834         "uid": "grafana"
5835       },
5836       "gridPos": {
5837         "h": 1,
5838         "w": 24,
5839         "x": 0,
5840         "y": 69
5841       },
5842       "id": 46,
5843       "panels": [
5844         {
5845           "datasource": {
5846             "type": "prometheus",
5847             "uid": "PBFA97CFB590B2093"
5848           },
5849           "description": "The number of messages produced converted to match the log.message.format.version.",
5850           "fieldConfig": {
5851             "defaults": {
5852               "color": {
5853                 "mode": "palette-classic"
5854               },
5855               "custom": {
5856                 "axisLabel": "",
5857                 "axisPlacement": "auto",
5858                 "barAlignment": 0,
5859                 "drawStyle": "line",
5860                 "fillOpacity": 10,
5861                 "gradientMode": "none",
5862                 "hideFrom": {
5863                   "legend": false,
5864                   "tooltip": false,
5865                   "viz": false
5866                 },
5867                 "lineInterpolation": "linear",
5868                 "lineWidth": 1,
5869                 "pointSize": 5,
5870                 "scaleDistribution": {
5871                   "type": "linear"
5872                 },
5873                 "showPoints": "never",
5874                 "spanNulls": true,
5875                 "stacking": {
5876                   "group": "A",
5877                   "mode": "none"
5878                 },
5879                 "thresholdsStyle": {
5880                   "mode": "off"
5881                 }
5882               },
5883               "links": [],
5884               "mappings": [],
5885               "min": 0,
5886               "thresholds": {
5887                 "mode": "absolute",
5888                 "steps": [
5889                   {
5890                     "color": "green"
5891                   },
5892                   {
5893                     "color": "red",
5894                     "value": 80
5895                   }
5896                 ]
5897               },
5898               "unit": "ops"
5899             },
5900             "overrides": []
5901           },
5902           "gridPos": {
5903             "h": 8,
5904             "w": 8,
5905             "x": 0,
5906             "y": 34
5907           },
5908           "id": 48,
5909           "options": {
5910             "legend": {
5911               "calcs": [
5912                 "mean",
5913                 "lastNotNull",
5914                 "max"
5915               ],
5916               "displayMode": "list",
5917               "placement": "bottom",
5918               "showLegend": true
5919             },
5920             "tooltip": {
5921               "mode": "single"
5922             }
5923           },
5924           "pluginVersion": "8.1.3",
5925           "targets": [
5926             {
5927               "datasource": {
5928                 "type": "prometheus",
5929                 "uid": "PBFA97CFB590B2093"
5930               },
5931               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
5932               "hide": false,
5933               "interval": "",
5934               "legendFormat": "{{topic}}",
5935               "refId": "A"
5936             }
5937           ],
5938           "title": "Number of produced message conversion",
5939           "type": "timeseries"
5940         },
5941         {
5942           "datasource": {
5943             "type": "prometheus",
5944             "uid": "PBFA97CFB590B2093"
5945           },
5946           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
5947           "fieldConfig": {
5948             "defaults": {
5949               "color": {
5950                 "mode": "palette-classic"
5951               },
5952               "custom": {
5953                 "axisLabel": "",
5954                 "axisPlacement": "auto",
5955                 "barAlignment": 0,
5956                 "drawStyle": "line",
5957                 "fillOpacity": 10,
5958                 "gradientMode": "none",
5959                 "hideFrom": {
5960                   "legend": false,
5961                   "tooltip": false,
5962                   "viz": false
5963                 },
5964                 "lineInterpolation": "linear",
5965                 "lineWidth": 1,
5966                 "pointSize": 5,
5967                 "scaleDistribution": {
5968                   "type": "linear"
5969                 },
5970                 "showPoints": "never",
5971                 "spanNulls": true,
5972                 "stacking": {
5973                   "group": "A",
5974                   "mode": "none"
5975                 },
5976                 "thresholdsStyle": {
5977                   "mode": "off"
5978                 }
5979               },
5980               "links": [],
5981               "mappings": [],
5982               "min": 0,
5983               "thresholds": {
5984                 "mode": "absolute",
5985                 "steps": [
5986                   {
5987                     "color": "green"
5988                   },
5989                   {
5990                     "color": "red",
5991                     "value": 80
5992                   }
5993                 ]
5994               },
5995               "unit": "ops"
5996             },
5997             "overrides": []
5998           },
5999           "gridPos": {
6000             "h": 8,
6001             "w": 8,
6002             "x": 8,
6003             "y": 34
6004           },
6005           "id": 51,
6006           "options": {
6007             "legend": {
6008               "calcs": [
6009                 "mean",
6010                 "lastNotNull",
6011                 "max"
6012               ],
6013               "displayMode": "list",
6014               "placement": "bottom",
6015               "showLegend": true
6016             },
6017             "tooltip": {
6018               "mode": "single"
6019             }
6020           },
6021           "pluginVersion": "8.1.3",
6022           "targets": [
6023             {
6024               "datasource": {
6025                 "type": "prometheus",
6026                 "uid": "PBFA97CFB590B2093"
6027               },
6028               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6029               "hide": false,
6030               "interval": "",
6031               "legendFormat": "{{topic}}",
6032               "refId": "A"
6033             }
6034           ],
6035           "title": "Number of consumed message conversion",
6036           "type": "timeseries"
6037         },
6038         {
6039           "datasource": {
6040             "type": "datasource",
6041             "uid": "grafana"
6042           },
6043           "description": "Number of connection per client version",
6044           "fieldConfig": {
6045             "defaults": {
6046               "color": {
6047                 "mode": "palette-classic"
6048               },
6049               "custom": {
6050                 "hideFrom": {
6051                   "legend": false,
6052                   "tooltip": false,
6053                   "viz": false
6054                 }
6055               },
6056               "decimals": 0,
6057               "mappings": [],
6058               "unit": "short"
6059             },
6060             "overrides": []
6061           },
6062           "gridPos": {
6063             "h": 8,
6064             "w": 8,
6065             "x": 16,
6066             "y": 34
6067           },
6068           "id": 96,
6069           "links": [],
6070           "options": {
6071             "legend": {
6072               "calcs": [],
6073               "displayMode": "table",
6074               "placement": "bottom",
6075               "showLegend": true,
6076               "values": [
6077                 "value"
6078               ]
6079             },
6080             "pieType": "pie",
6081             "reduceOptions": {
6082               "calcs": [
6083                 "lastNotNull"
6084               ],
6085               "fields": "",
6086               "values": false
6087             },
6088             "tooltip": {
6089               "mode": "single"
6090             }
6091           },
6092           "targets": [
6093             {
6094               "datasource": {
6095                 "type": "datasource",
6096                 "uid": "grafana"
6097               },
6098               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6099               "interval": "",
6100               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6101               "refId": "A"
6102             }
6103           ],
6104           "title": "Client version repartition",
6105           "type": "piechart"
6106         }
6107       ],
6108       "targets": [
6109         {
6110           "datasource": {
6111             "type": "datasource",
6112             "uid": "grafana"
6113           },
6114           "refId": "A"
6115         }
6116       ],
6117       "title": "Message Conversion",
6118       "type": "row"
6119     }
6120   ],
6121   "refresh": "5s",
6122   "schemaVersion": 38,
6123   "style": "dark",
6124   "tags": [],
6125   "templating": {
6126     "list": [
6127       {
6128         "current": {
6129           "selected": false,
6130           "text": "dev",
6131           "value": "dev"
6132         },
6133         "datasource": {
6134           "type": "prometheus",
6135           "uid": "PBFA97CFB590B2093"
6136         },
6137         "definition": "label_values(env)",
6138         "hide": 0,
6139         "includeAll": false,
6140         "label": "Environment",
6141         "multi": false,
6142         "name": "env",
6143         "options": [],
6144         "query": {
6145           "query": "label_values(env)",
6146           "refId": "Prometheus-env-Variable-Query"
6147         },
6148         "refresh": 1,
6149         "regex": "",
6150         "skipUrlSync": false,
6151         "sort": 0,
6152         "tagValuesQuery": "",
6153         "tagsQuery": "",
6154         "type": "query",
6155         "useTags": false
6156       },
6157       {
6158         "current": {
6159           "selected": true,
6160           "text": [
6161             "All"
6162           ],
6163           "value": [
6164             "$__all"
6165           ]
6166         },
6167         "datasource": {
6168           "type": "prometheus",
6169           "uid": "PBFA97CFB590B2093"
6170         },
6171         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6172         "hide": 0,
6173         "includeAll": true,
6174         "label": "Instance",
6175         "multi": true,
6176         "name": "instance",
6177         "options": [],
6178         "query": {
6179           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6180           "refId": "Prometheus-instance-Variable-Query"
6181         },
6182         "refresh": 1,
6183         "regex": "",
6184         "skipUrlSync": false,
6185         "sort": 0,
6186         "tagValuesQuery": "",
6187         "tagsQuery": "",
6188         "type": "query",
6189         "useTags": false
6190       },
6191       {
6192         "allValue": "",
6193         "current": {
6194           "selected": true,
6195           "text": [
6196             "0.95"
6197           ],
6198           "value": [
6199             "0.95"
6200           ]
6201         },
6202         "datasource": {
6203           "type": "prometheus",
6204           "uid": "PBFA97CFB590B2093"
6205         },
6206         "definition": "label_values(quantile)",
6207         "hide": 0,
6208         "includeAll": true,
6209         "label": "Percentile",
6210         "multi": true,
6211         "name": "percentile",
6212         "options": [],
6213         "query": {
6214           "query": "label_values(quantile)",
6215           "refId": "Prometheus-percentile-Variable-Query"
6216         },
6217         "refresh": 1,
6218         "regex": "",
6219         "skipUrlSync": false,
6220         "sort": 0,
6221         "tagValuesQuery": "",
6222         "tagsQuery": "",
6223         "type": "query",
6224         "useTags": false
6225       }
6226     ]
6227   },
6228   "time": {
6229     "from": "now-5m",
6230     "to": "now"
6231   },
6232   "timepicker": {
6233     "refresh_intervals": [
6234       "5s",
6235       "10s",
6236       "30s",
6237       "1m",
6238       "5m",
6239       "15m",
6240       "30m",
6241       "1h",
6242       "2h",
6243       "1d"
6244     ],
6245     "time_options": [
6246       "5m",
6247       "15m",
6248       "1h",
6249       "6h",
6250       "12h",
6251       "24h",
6252       "2d",
6253       "7d",
6254       "30d"
6255     ]
6256   },
6257   "timezone": "browser",
6258   "title": "Kafka cluster",
6259   "uid": "qu-QZdfZz",
6260   "version": 1,
6261   "weekStart": ""
6262 }