juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of Brokers Online",
  64       "fieldConfig": {
  65         "defaults": {
  66           "color": {
  67             "mode": "thresholds"
  68           },
  69           "mappings": [
  70             {
  71               "options": {
  72                 "match": "null",
  73                 "result": {
  74                   "text": "N/A"
  75                 }
  76               },
  77               "type": "special"
  78             }
  79           ],
  80           "thresholds": {
  81             "mode": "absolute",
  82             "steps": [
  83               {
  84                 "color": "#d44a3a",
  85                 "value": null
  86               },
  87               {
  88                 "color": "rgba(237, 129, 40, 0.89)",
  89                 "value": 0
  90               },
  91               {
  92                 "color": "#299c46",
  93                 "value": 2
  94               }
  95             ]
  96           },
  97           "unit": "none"
  98         },
  99         "overrides": []
 100       },
 101       "gridPos": {
 102         "h": 4,
 103         "w": 4,
 104         "x": 0,
 105         "y": 1
 106       },
 107       "id": 14,
 108       "links": [],
 109       "maxDataPoints": 100,
 110       "options": {
 111         "colorMode": "value",
 112         "graphMode": "none",
 113         "justifyMode": "auto",
 114         "orientation": "horizontal",
 115         "reduceOptions": {
 116           "calcs": [
 117             "lastNotNull"
 118           ],
 119           "fields": "",
 120           "values": false
 121         },
 122         "text": {},
 123         "textMode": "auto"
 124       },
 125       "pluginVersion": "10.0.3",
 126       "repeatDirection": "h",
 127       "targets": [
 128         {
 129           "datasource": {
 130             "type": "prometheus",
 131             "uid": "PBFA97CFB590B2093"
 132           },
 133           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 134           "format": "time_series",
 135           "instant": true,
 136           "interval": "",
 137           "intervalFactor": 1,
 138           "legendFormat": "",
 139           "refId": "A"
 140         }
 141       ],
 142       "title": "Brokers Online",
 143       "type": "stat"
 144     },
 145     {
 146       "datasource": {
 147         "type": "prometheus",
 148         "uid": "PBFA97CFB590B2093"
 149       },
 150       "description": "Number of active controllers in the cluster.",
 151       "fieldConfig": {
 152         "defaults": {
 153           "mappings": [
 154             {
 155               "options": {
 156                 "match": "null",
 157                 "result": {
 158                   "text": "N/A"
 159                 }
 160               },
 161               "type": "special"
 162             }
 163           ],
 164           "thresholds": {
 165             "mode": "absolute",
 166             "steps": [
 167               {
 168                 "color": "#299c46",
 169                 "value": null
 170               },
 171               {
 172                 "color": "#e5ac0e",
 173                 "value": 2
 174               },
 175               {
 176                 "color": "#bf1b00"
 177               }
 178             ]
 179           },
 180           "unit": "none"
 181         },
 182         "overrides": []
 183       },
 184       "gridPos": {
 185         "h": 4,
 186         "w": 5,
 187         "x": 4,
 188         "y": 1
 189       },
 190       "id": 12,
 191       "links": [],
 192       "maxDataPoints": 100,
 193       "options": {
 194         "colorMode": "value",
 195         "graphMode": "none",
 196         "justifyMode": "auto",
 197         "orientation": "vertical",
 198         "reduceOptions": {
 199           "calcs": [
 200             "lastNotNull"
 201           ],
 202           "fields": "",
 203           "values": false
 204         },
 205         "text": {},
 206         "textMode": "value_and_name"
 207       },
 208       "pluginVersion": "10.0.3",
 209       "targets": [
 210         {
 211           "datasource": {
 212             "type": "prometheus",
 213             "uid": "PBFA97CFB590B2093"
 214           },
 215           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 216           "format": "time_series",
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 1,
 220           "legendFormat": "{{instance}}",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Active Controllers",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Partitions that are online",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#d44a3a",
 254                 "value": null
 255               },
 256               {
 257                 "color": "rgba(237, 129, 40, 0.89)",
 258                 "value": 0
 259               },
 260               {
 261                 "color": "#299c46",
 262                 "value": 0
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 6,
 273         "x": 9,
 274         "y": 1
 275       },
 276       "id": 18,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Online Partitions",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 15,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "description": "Unclean leader election rate",
 404       "fieldConfig": {
 405         "defaults": {
 406           "color": {
 407             "mode": "thresholds"
 408           },
 409           "mappings": [
 410             {
 411               "options": {
 412                 "match": "null",
 413                 "result": {
 414                   "text": "N/A"
 415                 }
 416               },
 417               "type": "special"
 418             }
 419           ],
 420           "thresholds": {
 421             "mode": "absolute",
 422             "steps": [
 423               {
 424                 "color": "#299c46",
 425                 "value": null
 426               },
 427               {
 428                 "color": "rgba(237, 129, 40, 0.89)",
 429                 "value": 2
 430               },
 431               {
 432                 "color": "#d44a3a"
 433               }
 434             ]
 435           },
 436           "unit": "none"
 437         },
 438         "overrides": []
 439       },
 440       "gridPos": {
 441         "h": 4,
 442         "w": 4,
 443         "x": 19,
 444         "y": 1
 445       },
 446       "id": 16,
 447       "links": [],
 448       "maxDataPoints": 100,
 449       "options": {
 450         "colorMode": "value",
 451         "graphMode": "none",
 452         "justifyMode": "auto",
 453         "orientation": "horizontal",
 454         "reduceOptions": {
 455           "calcs": [
 456             "lastNotNull"
 457           ],
 458           "fields": "",
 459           "values": false
 460         },
 461         "text": {},
 462         "textMode": "auto"
 463       },
 464       "pluginVersion": "10.0.3",
 465       "targets": [
 466         {
 467           "datasource": {
 468             "type": "prometheus",
 469             "uid": "PBFA97CFB590B2093"
 470           },
 471           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 472           "format": "time_series",
 473           "instant": true,
 474           "interval": "",
 475           "intervalFactor": 1,
 476           "legendFormat": "",
 477           "refId": "A"
 478         }
 479       ],
 480       "title": "Unclean Leader Election Rate",
 481       "type": "stat"
 482     },
 483     {
 484       "datasource": {
 485         "type": "prometheus",
 486         "uid": "PBFA97CFB590B2093"
 487       },
 488       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 489       "fieldConfig": {
 490         "defaults": {
 491           "color": {
 492             "mode": "thresholds"
 493           },
 494           "mappings": [
 495             {
 496               "options": {
 497                 "match": "null",
 498                 "result": {
 499                   "text": "N/A"
 500                 }
 501               },
 502               "type": "special"
 503             }
 504           ],
 505           "thresholds": {
 506             "mode": "absolute",
 507             "steps": [
 508               {
 509                 "color": "#508642",
 510                 "value": null
 511               },
 512               {
 513                 "color": "rgba(237, 129, 40, 0.89)",
 514                 "value": 1
 515               },
 516               {
 517                 "color": "#bf1b00",
 518                 "value": 5
 519               }
 520             ]
 521           },
 522           "unit": "none"
 523         },
 524         "overrides": []
 525       },
 526       "gridPos": {
 527         "h": 4,
 528         "w": 4,
 529         "x": 0,
 530         "y": 5
 531       },
 532       "id": 20,
 533       "links": [],
 534       "maxDataPoints": 100,
 535       "options": {
 536         "colorMode": "value",
 537         "graphMode": "area",
 538         "justifyMode": "auto",
 539         "orientation": "horizontal",
 540         "reduceOptions": {
 541           "calcs": [
 542             "lastNotNull"
 543           ],
 544           "fields": "",
 545           "values": false
 546         },
 547         "text": {},
 548         "textMode": "auto"
 549       },
 550       "pluginVersion": "10.0.3",
 551       "targets": [
 552         {
 553           "datasource": {
 554             "type": "prometheus",
 555             "uid": "PBFA97CFB590B2093"
 556           },
 557           "editorMode": "code",
 558           "exemplar": false,
 559           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 560           "format": "time_series",
 561           "hide": false,
 562           "instant": false,
 563           "interval": "",
 564           "intervalFactor": 2,
 565           "legendFormat": "",
 566           "range": true,
 567           "refId": "A"
 568         }
 569       ],
 570       "title": "Under Replicated Partitions",
 571       "type": "stat"
 572     },
 573     {
 574       "datasource": {
 575         "type": "prometheus",
 576         "uid": "PBFA97CFB590B2093"
 577       },
 578       "description": "",
 579       "fieldConfig": {
 580         "defaults": {
 581           "color": {
 582             "mode": "palette-classic"
 583           },
 584           "custom": {
 585             "axisCenteredZero": false,
 586             "axisColorMode": "text",
 587             "axisLabel": "",
 588             "axisPlacement": "auto",
 589             "barAlignment": 0,
 590             "drawStyle": "line",
 591             "fillOpacity": 10,
 592             "gradientMode": "none",
 593             "hideFrom": {
 594               "legend": false,
 595               "tooltip": false,
 596               "viz": false
 597             },
 598             "lineInterpolation": "linear",
 599             "lineWidth": 1,
 600             "pointSize": 5,
 601             "scaleDistribution": {
 602               "type": "linear"
 603             },
 604             "showPoints": "never",
 605             "spanNulls": true,
 606             "stacking": {
 607               "group": "A",
 608               "mode": "none"
 609             },
 610             "thresholdsStyle": {
 611               "mode": "off"
 612             }
 613           },
 614           "decimals": 0,
 615           "links": [],
 616           "mappings": [
 617             {
 618               "options": {
 619                 "null": {
 620                   "index": 0,
 621                   "text": "N/A"
 622                 }
 623               },
 624               "type": "value"
 625             }
 626           ],
 627           "min": 0,
 628           "thresholds": {
 629             "mode": "absolute",
 630             "steps": [
 631               {
 632                 "color": "green",
 633                 "value": null
 634               },
 635               {
 636                 "color": "red",
 637                 "value": 80
 638               }
 639             ]
 640           },
 641           "unit": "partitions"
 642         },
 643         "overrides": []
 644       },
 645       "gridPos": {
 646         "h": 12,
 647         "w": 11,
 648         "x": 4,
 649         "y": 5
 650       },
 651       "id": 123,
 652       "options": {
 653         "legend": {
 654           "calcs": [
 655             "last",
 656             "max"
 657           ],
 658           "displayMode": "table",
 659           "placement": "bottom",
 660           "showLegend": true
 661         },
 662         "tooltip": {
 663           "mode": "single",
 664           "sort": "none"
 665         }
 666       },
 667       "pluginVersion": "8.1.3",
 668       "targets": [
 669         {
 670           "datasource": {
 671             "type": "prometheus",
 672             "uid": "PBFA97CFB590B2093"
 673           },
 674           "editorMode": "code",
 675           "exemplar": false,
 676           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 677           "hide": false,
 678           "interval": "",
 679           "legendFormat": "{{instance}}",
 680           "range": true,
 681           "refId": "A"
 682         }
 683       ],
 684       "title": "Underreplicated Partitions",
 685       "type": "timeseries"
 686     },
 687     {
 688       "datasource": {
 689         "type": "prometheus",
 690         "uid": "PBFA97CFB590B2093"
 691       },
 692       "description": "",
 693       "fieldConfig": {
 694         "defaults": {
 695           "color": {
 696             "mode": "palette-classic"
 697           },
 698           "custom": {
 699             "hideFrom": {
 700               "legend": false,
 701               "tooltip": false,
 702               "viz": false
 703             }
 704           },
 705           "decimals": 0,
 706           "links": [],
 707           "mappings": [
 708             {
 709               "options": {
 710                 "null": {
 711                   "index": 0,
 712                   "text": "0"
 713                 }
 714               },
 715               "type": "value"
 716             }
 717           ],
 718           "min": 0,
 719           "unit": "partitions"
 720         },
 721         "overrides": []
 722       },
 723       "gridPos": {
 724         "h": 12,
 725         "w": 8,
 726         "x": 15,
 727         "y": 5
 728       },
 729       "id": 124,
 730       "options": {
 731         "displayLabels": [
 732           "percent"
 733         ],
 734         "legend": {
 735           "displayMode": "list",
 736           "placement": "right",
 737           "showLegend": true,
 738           "values": [
 739             "value"
 740           ]
 741         },
 742         "pieType": "pie",
 743         "reduceOptions": {
 744           "calcs": [
 745             "last"
 746           ],
 747           "fields": "",
 748           "values": false
 749         },
 750         "tooltip": {
 751           "mode": "single",
 752           "sort": "none"
 753         }
 754       },
 755       "pluginVersion": "8.1.3",
 756       "targets": [
 757         {
 758           "datasource": {
 759             "type": "prometheus",
 760             "uid": "PBFA97CFB590B2093"
 761           },
 762           "editorMode": "code",
 763           "exemplar": false,
 764           "expr": "kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 765           "hide": false,
 766           "interval": "",
 767           "legendFormat": "{{instance}}",
 768           "range": true,
 769           "refId": "A"
 770         }
 771       ],
 772       "title": "Leader Count",
 773       "type": "piechart"
 774     },
 775     {
 776       "datasource": {
 777         "type": "prometheus",
 778         "uid": "PBFA97CFB590B2093"
 779       },
 780       "description": "Number of partitions under min insync replicas.",
 781       "fieldConfig": {
 782         "defaults": {
 783           "color": {
 784             "mode": "thresholds"
 785           },
 786           "mappings": [
 787             {
 788               "options": {
 789                 "match": "null",
 790                 "result": {
 791                   "text": "N/A"
 792                 }
 793               },
 794               "type": "special"
 795             }
 796           ],
 797           "thresholds": {
 798             "mode": "absolute",
 799             "steps": [
 800               {
 801                 "color": "#508642",
 802                 "value": null
 803               },
 804               {
 805                 "color": "rgba(237, 129, 40, 0.89)",
 806                 "value": 1
 807               },
 808               {
 809                 "color": "#bf1b00",
 810                 "value": 5
 811               }
 812             ]
 813           },
 814           "unit": "none"
 815         },
 816         "overrides": []
 817       },
 818       "gridPos": {
 819         "h": 4,
 820         "w": 4,
 821         "x": 0,
 822         "y": 9
 823       },
 824       "id": 32,
 825       "links": [],
 826       "maxDataPoints": 100,
 827       "options": {
 828         "colorMode": "value",
 829         "graphMode": "area",
 830         "justifyMode": "auto",
 831         "orientation": "horizontal",
 832         "reduceOptions": {
 833           "calcs": [
 834             "lastNotNull"
 835           ],
 836           "fields": "",
 837           "values": false
 838         },
 839         "text": {},
 840         "textMode": "auto"
 841       },
 842       "pluginVersion": "10.0.3",
 843       "targets": [
 844         {
 845           "datasource": {
 846             "type": "prometheus",
 847             "uid": "PBFA97CFB590B2093"
 848           },
 849           "editorMode": "code",
 850           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 851           "format": "time_series",
 852           "hide": false,
 853           "instant": false,
 854           "interval": "",
 855           "intervalFactor": 2,
 856           "legendFormat": "",
 857           "range": true,
 858           "refId": "A"
 859         }
 860       ],
 861       "title": "Under Min ISR Partitions",
 862       "type": "stat"
 863     },
 864     {
 865       "datasource": {
 866         "type": "prometheus",
 867         "uid": "PBFA97CFB590B2093"
 868       },
 869       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 870       "fieldConfig": {
 871         "defaults": {
 872           "color": {
 873             "mode": "thresholds"
 874           },
 875           "mappings": [
 876             {
 877               "options": {
 878                 "match": "null",
 879                 "result": {
 880                   "text": "N/A"
 881                 }
 882               },
 883               "type": "special"
 884             }
 885           ],
 886           "thresholds": {
 887             "mode": "absolute",
 888             "steps": [
 889               {
 890                 "color": "#508642",
 891                 "value": null
 892               },
 893               {
 894                 "color": "#ef843c",
 895                 "value": 1
 896               },
 897               {
 898                 "color": "#bf1b00",
 899                 "value": 1
 900               }
 901             ]
 902           },
 903           "unit": "none"
 904         },
 905         "overrides": []
 906       },
 907       "gridPos": {
 908         "h": 4,
 909         "w": 4,
 910         "x": 0,
 911         "y": 13
 912       },
 913       "id": 22,
 914       "links": [],
 915       "maxDataPoints": 100,
 916       "options": {
 917         "colorMode": "value",
 918         "graphMode": "area",
 919         "justifyMode": "auto",
 920         "orientation": "horizontal",
 921         "reduceOptions": {
 922           "calcs": [
 923             "lastNotNull"
 924           ],
 925           "fields": "",
 926           "values": false
 927         },
 928         "text": {},
 929         "textMode": "auto"
 930       },
 931       "pluginVersion": "10.0.3",
 932       "targets": [
 933         {
 934           "datasource": {
 935             "type": "prometheus",
 936             "uid": "PBFA97CFB590B2093"
 937           },
 938           "editorMode": "code",
 939           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 940           "format": "time_series",
 941           "instant": false,
 942           "interval": "",
 943           "intervalFactor": 1,
 944           "legendFormat": "",
 945           "range": true,
 946           "refId": "A"
 947         }
 948       ],
 949       "title": "Offline Partitions Count",
 950       "type": "stat"
 951     },
 952     {
 953       "collapsed": true,
 954       "datasource": {
 955         "type": "datasource",
 956         "uid": "grafana"
 957       },
 958       "gridPos": {
 959         "h": 1,
 960         "w": 24,
 961         "x": 0,
 962         "y": 17
 963       },
 964       "id": 40,
 965       "panels": [
 966         {
 967           "datasource": {
 968             "type": "prometheus",
 969             "uid": "PBFA97CFB590B2093"
 970           },
 971           "fieldConfig": {
 972             "defaults": {
 973               "color": {
 974                 "mode": "palette-classic"
 975               },
 976               "custom": {
 977                 "axisCenteredZero": false,
 978                 "axisColorMode": "text",
 979                 "axisLabel": "Cores",
 980                 "axisPlacement": "auto",
 981                 "barAlignment": 0,
 982                 "drawStyle": "line",
 983                 "fillOpacity": 10,
 984                 "gradientMode": "none",
 985                 "hideFrom": {
 986                   "legend": false,
 987                   "tooltip": false,
 988                   "viz": false
 989                 },
 990                 "lineInterpolation": "linear",
 991                 "lineWidth": 2,
 992                 "pointSize": 5,
 993                 "scaleDistribution": {
 994                   "type": "linear"
 995                 },
 996                 "showPoints": "never",
 997                 "spanNulls": false,
 998                 "stacking": {
 999                   "group": "A",
1000                   "mode": "none"
1001                 },
1002                 "thresholdsStyle": {
1003                   "mode": "off"
1004                 }
1005               },
1006               "links": [],
1007               "mappings": [],
1008               "thresholds": {
1009                 "mode": "absolute",
1010                 "steps": [
1011                   {
1012                     "color": "green"
1013                   },
1014                   {
1015                     "color": "red",
1016                     "value": 80
1017                   }
1018                 ]
1019               },
1020               "unit": "percent"
1021             },
1022             "overrides": [
1023               {
1024                 "matcher": {
1025                   "id": "byName",
1026                   "options": "localhost:7071"
1027                 },
1028                 "properties": [
1029                   {
1030                     "id": "color",
1031                     "value": {
1032                       "fixedColor": "#629E51",
1033                       "mode": "fixed"
1034                     }
1035                   }
1036                 ]
1037               }
1038             ]
1039           },
1040           "gridPos": {
1041             "h": 12,
1042             "w": 8,
1043             "x": 0,
1044             "y": 19
1045           },
1046           "id": 27,
1047           "links": [],
1048           "options": {
1049             "legend": {
1050               "calcs": [
1051                 "mean",
1052                 "lastNotNull",
1053                 "max"
1054               ],
1055               "displayMode": "table",
1056               "placement": "bottom",
1057               "showLegend": true
1058             },
1059             "tooltip": {
1060               "mode": "single",
1061               "sort": "none"
1062             }
1063           },
1064           "pluginVersion": "8.1.3",
1065           "targets": [
1066             {
1067               "datasource": {
1068                 "type": "prometheus",
1069                 "uid": "PBFA97CFB590B2093"
1070               },
1071               "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1072               "format": "time_series",
1073               "interval": "",
1074               "intervalFactor": 2,
1075               "legendFormat": "{{instance}}",
1076               "metric": "process_cpu_secondspersec",
1077               "refId": "A",
1078               "step": 4
1079             }
1080           ],
1081           "title": "CPU Usage",
1082           "type": "timeseries"
1083         },
1084         {
1085           "datasource": {
1086             "type": "prometheus",
1087             "uid": "PBFA97CFB590B2093"
1088           },
1089           "fieldConfig": {
1090             "defaults": {
1091               "color": {
1092                 "mode": "palette-classic"
1093               },
1094               "custom": {
1095                 "axisCenteredZero": false,
1096                 "axisColorMode": "text",
1097                 "axisLabel": "Memory",
1098                 "axisPlacement": "auto",
1099                 "barAlignment": 0,
1100                 "drawStyle": "line",
1101                 "fillOpacity": 10,
1102                 "gradientMode": "none",
1103                 "hideFrom": {
1104                   "legend": false,
1105                   "tooltip": false,
1106                   "viz": false
1107                 },
1108                 "lineInterpolation": "linear",
1109                 "lineWidth": 2,
1110                 "pointSize": 5,
1111                 "scaleDistribution": {
1112                   "type": "linear"
1113                 },
1114                 "showPoints": "never",
1115                 "spanNulls": false,
1116                 "stacking": {
1117                   "group": "A",
1118                   "mode": "none"
1119                 },
1120                 "thresholdsStyle": {
1121                   "mode": "off"
1122                 }
1123               },
1124               "links": [],
1125               "mappings": [],
1126               "thresholds": {
1127                 "mode": "absolute",
1128                 "steps": [
1129                   {
1130                     "color": "green"
1131                   },
1132                   {
1133                     "color": "red",
1134                     "value": 80
1135                   }
1136                 ]
1137               },
1138               "unit": "bytes"
1139             },
1140             "overrides": [
1141               {
1142                 "matcher": {
1143                   "id": "byName",
1144                   "options": "localhost:7071"
1145                 },
1146                 "properties": [
1147                   {
1148                     "id": "color",
1149                     "value": {
1150                       "fixedColor": "#BA43A9",
1151                       "mode": "fixed"
1152                     }
1153                   }
1154                 ]
1155               },
1156               {
1157                 "__systemRef": "hideSeriesFrom",
1158                 "matcher": {
1159                   "id": "byNames",
1160                   "options": {
1161                     "mode": "exclude",
1162                     "names": [
1163                       "kafka-2:80"
1164                     ],
1165                     "prefix": "All except:",
1166                     "readOnly": true
1167                   }
1168                 },
1169                 "properties": [
1170                   {
1171                     "id": "custom.hideFrom",
1172                     "value": {
1173                       "legend": false,
1174                       "tooltip": false,
1175                       "viz": true
1176                     }
1177                   }
1178                 ]
1179               }
1180             ]
1181           },
1182           "gridPos": {
1183             "h": 12,
1184             "w": 8,
1185             "x": 8,
1186             "y": 19
1187           },
1188           "id": 2,
1189           "links": [],
1190           "options": {
1191             "legend": {
1192               "calcs": [
1193                 "mean",
1194                 "lastNotNull",
1195                 "max"
1196               ],
1197               "displayMode": "table",
1198               "placement": "bottom",
1199               "showLegend": true
1200             },
1201             "tooltip": {
1202               "mode": "single",
1203               "sort": "none"
1204             }
1205           },
1206           "pluginVersion": "8.1.3",
1207           "targets": [
1208             {
1209               "datasource": {
1210                 "type": "prometheus",
1211                 "uid": "PBFA97CFB590B2093"
1212               },
1213               "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1214               "interval": "",
1215               "intervalFactor": 2,
1216               "legendFormat": "{{instance}}",
1217               "metric": "jvm_memory_bytes_used",
1218               "refId": "A",
1219               "step": 4
1220             },
1221             {
1222               "datasource": {
1223                 "type": "prometheus",
1224                 "uid": "PBFA97CFB590B2093"
1225               },
1226               "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1227               "interval": "",
1228               "legendFormat": "{{instance}}",
1229               "refId": "B"
1230             }
1231           ],
1232           "title": "JVM Memory Used",
1233           "type": "timeseries"
1234         },
1235         {
1236           "datasource": {
1237             "type": "prometheus",
1238             "uid": "PBFA97CFB590B2093"
1239           },
1240           "fieldConfig": {
1241             "defaults": {
1242               "color": {
1243                 "mode": "palette-classic"
1244               },
1245               "custom": {
1246                 "axisCenteredZero": false,
1247                 "axisColorMode": "text",
1248                 "axisLabel": "% time in GC",
1249                 "axisPlacement": "auto",
1250                 "barAlignment": 0,
1251                 "drawStyle": "line",
1252                 "fillOpacity": 10,
1253                 "gradientMode": "none",
1254                 "hideFrom": {
1255                   "legend": false,
1256                   "tooltip": false,
1257                   "viz": false
1258                 },
1259                 "lineInterpolation": "linear",
1260                 "lineWidth": 2,
1261                 "pointSize": 5,
1262                 "scaleDistribution": {
1263                   "type": "linear"
1264                 },
1265                 "showPoints": "never",
1266                 "spanNulls": false,
1267                 "stacking": {
1268                   "group": "A",
1269                   "mode": "none"
1270                 },
1271                 "thresholdsStyle": {
1272                   "mode": "off"
1273                 }
1274               },
1275               "links": [],
1276               "mappings": [],
1277               "thresholds": {
1278                 "mode": "absolute",
1279                 "steps": [
1280                   {
1281                     "color": "green"
1282                   },
1283                   {
1284                     "color": "red",
1285                     "value": 80
1286                   }
1287                 ]
1288               },
1289               "unit": "percentunit"
1290             },
1291             "overrides": [
1292               {
1293                 "matcher": {
1294                   "id": "byName",
1295                   "options": "localhost:7071"
1296                 },
1297                 "properties": [
1298                   {
1299                     "id": "color",
1300                     "value": {
1301                       "fixedColor": "#890F02",
1302                       "mode": "fixed"
1303                     }
1304                   }
1305                 ]
1306               }
1307             ]
1308           },
1309           "gridPos": {
1310             "h": 12,
1311             "w": 8,
1312             "x": 16,
1313             "y": 19
1314           },
1315           "id": 3,
1316           "links": [],
1317           "options": {
1318             "legend": {
1319               "calcs": [
1320                 "mean",
1321                 "lastNotNull",
1322                 "max"
1323               ],
1324               "displayMode": "table",
1325               "placement": "bottom",
1326               "showLegend": true
1327             },
1328             "tooltip": {
1329               "mode": "single",
1330               "sort": "none"
1331             }
1332           },
1333           "pluginVersion": "8.1.3",
1334           "targets": [
1335             {
1336               "datasource": {
1337                 "type": "prometheus",
1338                 "uid": "PBFA97CFB590B2093"
1339               },
1340               "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1341               "interval": "",
1342               "intervalFactor": 2,
1343               "legendFormat": "{{instance}}",
1344               "metric": "jvm_gc_collection_seconds_sum",
1345               "refId": "A",
1346               "step": 4
1347             }
1348           ],
1349           "title": "Time spent in GC",
1350           "type": "timeseries"
1351         }
1352       ],
1353       "targets": [
1354         {
1355           "datasource": {
1356             "type": "datasource",
1357             "uid": "grafana"
1358           },
1359           "refId": "A"
1360         }
1361       ],
1362       "title": "System",
1363       "type": "row"
1364     },
1365     {
1366       "collapsed": false,
1367       "datasource": {
1368         "type": "datasource",
1369         "uid": "grafana"
1370       },
1371       "gridPos": {
1372         "h": 1,
1373         "w": 24,
1374         "x": 0,
1375         "y": 18
1376       },
1377       "id": 31,
1378       "panels": [],
1379       "targets": [
1380         {
1381           "datasource": {
1382             "type": "datasource",
1383             "uid": "grafana"
1384           },
1385           "refId": "A"
1386         }
1387       ],
1388       "title": "Throughput & Request rate",
1389       "type": "row"
1390     },
1391     {
1392       "datasource": {
1393         "type": "prometheus",
1394         "uid": "PBFA97CFB590B2093"
1395       },
1396       "fieldConfig": {
1397         "defaults": {
1398           "color": {
1399             "mode": "palette-classic"
1400           },
1401           "custom": {
1402             "axisCenteredZero": false,
1403             "axisColorMode": "text",
1404             "axisLabel": "Bytes/s",
1405             "axisPlacement": "auto",
1406             "barAlignment": 0,
1407             "drawStyle": "line",
1408             "fillOpacity": 10,
1409             "gradientMode": "none",
1410             "hideFrom": {
1411               "legend": false,
1412               "tooltip": false,
1413               "viz": false
1414             },
1415             "lineInterpolation": "linear",
1416             "lineWidth": 2,
1417             "pointSize": 5,
1418             "scaleDistribution": {
1419               "type": "linear"
1420             },
1421             "showPoints": "never",
1422             "spanNulls": false,
1423             "stacking": {
1424               "group": "A",
1425               "mode": "none"
1426             },
1427             "thresholdsStyle": {
1428               "mode": "off"
1429             }
1430           },
1431           "links": [],
1432           "mappings": [],
1433           "thresholds": {
1434             "mode": "absolute",
1435             "steps": [
1436               {
1437                 "color": "green",
1438                 "value": null
1439               },
1440               {
1441                 "color": "red",
1442                 "value": 80
1443               }
1444             ]
1445           },
1446           "unit": "Bps"
1447         },
1448         "overrides": []
1449       },
1450       "gridPos": {
1451         "h": 8,
1452         "w": 12,
1453         "x": 0,
1454         "y": 19
1455       },
1456       "id": 84,
1457       "links": [],
1458       "options": {
1459         "legend": {
1460           "calcs": [
1461             "mean",
1462             "lastNotNull",
1463             "max"
1464           ],
1465           "displayMode": "list",
1466           "placement": "bottom",
1467           "showLegend": true
1468         },
1469         "tooltip": {
1470           "mode": "single",
1471           "sort": "none"
1472         }
1473       },
1474       "pluginVersion": "8.1.3",
1475       "targets": [
1476         {
1477           "datasource": {
1478             "type": "prometheus",
1479             "uid": "PBFA97CFB590B2093"
1480           },
1481           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1482           "format": "time_series",
1483           "interval": "",
1484           "intervalFactor": 2,
1485           "legendFormat": "Bytes in",
1486           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1487           "refId": "A",
1488           "step": 4
1489         },
1490         {
1491           "datasource": {
1492             "type": "prometheus",
1493             "uid": "PBFA97CFB590B2093"
1494           },
1495           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
1496           "format": "time_series",
1497           "hide": false,
1498           "interval": "",
1499           "intervalFactor": 2,
1500           "legendFormat": "Bytes out",
1501           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
1502           "refId": "B",
1503           "step": 4
1504         }
1505       ],
1506       "title": "Broker network throughput",
1507       "type": "timeseries"
1508     },
1509     {
1510       "datasource": {
1511         "type": "prometheus",
1512         "uid": "PBFA97CFB590B2093"
1513       },
1514       "description": "Produce request rate.",
1515       "fieldConfig": {
1516         "defaults": {
1517           "mappings": [
1518             {
1519               "options": {
1520                 "match": "null",
1521                 "result": {
1522                   "text": "N/A"
1523                 }
1524               },
1525               "type": "special"
1526             }
1527           ],
1528           "thresholds": {
1529             "mode": "absolute",
1530             "steps": [
1531               {
1532                 "color": "green",
1533                 "value": null
1534               }
1535             ]
1536           },
1537           "unit": "none"
1538         },
1539         "overrides": []
1540       },
1541       "gridPos": {
1542         "h": 4,
1543         "w": 4,
1544         "x": 12,
1545         "y": 19
1546       },
1547       "id": 93,
1548       "links": [],
1549       "maxDataPoints": 100,
1550       "options": {
1551         "colorMode": "value",
1552         "graphMode": "area",
1553         "justifyMode": "auto",
1554         "orientation": "horizontal",
1555         "reduceOptions": {
1556           "calcs": [
1557             "lastNotNull"
1558           ],
1559           "fields": "",
1560           "values": false
1561         },
1562         "text": {},
1563         "textMode": "auto"
1564       },
1565       "pluginVersion": "10.0.3",
1566       "targets": [
1567         {
1568           "datasource": {
1569             "type": "prometheus",
1570             "uid": "PBFA97CFB590B2093"
1571           },
1572           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1573           "interval": "",
1574           "legendFormat": "",
1575           "refId": "A"
1576         }
1577       ],
1578       "title": "All Request Per Sec",
1579       "type": "stat"
1580     },
1581     {
1582       "datasource": {
1583         "type": "prometheus",
1584         "uid": "PBFA97CFB590B2093"
1585       },
1586       "description": "Fetch request rate.",
1587       "fieldConfig": {
1588         "defaults": {
1589           "mappings": [
1590             {
1591               "options": {
1592                 "match": "null",
1593                 "result": {
1594                   "text": "N/A"
1595                 }
1596               },
1597               "type": "special"
1598             }
1599           ],
1600           "thresholds": {
1601             "mode": "absolute",
1602             "steps": [
1603               {
1604                 "color": "green",
1605                 "value": null
1606               }
1607             ]
1608           },
1609           "unit": "none"
1610         },
1611         "overrides": []
1612       },
1613       "gridPos": {
1614         "h": 4,
1615         "w": 4,
1616         "x": 16,
1617         "y": 19
1618       },
1619       "id": 37,
1620       "links": [],
1621       "maxDataPoints": 100,
1622       "options": {
1623         "colorMode": "value",
1624         "graphMode": "area",
1625         "justifyMode": "auto",
1626         "orientation": "horizontal",
1627         "reduceOptions": {
1628           "calcs": [
1629             "lastNotNull"
1630           ],
1631           "fields": "",
1632           "values": false
1633         },
1634         "text": {},
1635         "textMode": "auto"
1636       },
1637       "pluginVersion": "10.0.3",
1638       "targets": [
1639         {
1640           "datasource": {
1641             "type": "prometheus",
1642             "uid": "PBFA97CFB590B2093"
1643           },
1644           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1645           "interval": "",
1646           "legendFormat": "",
1647           "refId": "A"
1648         }
1649       ],
1650       "title": "Consumer Fetch Request Per Sec",
1651       "type": "stat"
1652     },
1653     {
1654       "datasource": {
1655         "type": "prometheus",
1656         "uid": "PBFA97CFB590B2093"
1657       },
1658       "description": "Produce request rate.",
1659       "fieldConfig": {
1660         "defaults": {
1661           "mappings": [
1662             {
1663               "options": {
1664                 "match": "null",
1665                 "result": {
1666                   "text": "N/A"
1667                 }
1668               },
1669               "type": "special"
1670             }
1671           ],
1672           "thresholds": {
1673             "mode": "absolute",
1674             "steps": [
1675               {
1676                 "color": "green",
1677                 "value": null
1678               }
1679             ]
1680           },
1681           "unit": "none"
1682         },
1683         "overrides": []
1684       },
1685       "gridPos": {
1686         "h": 4,
1687         "w": 4,
1688         "x": 20,
1689         "y": 19
1690       },
1691       "id": 35,
1692       "links": [],
1693       "maxDataPoints": 100,
1694       "options": {
1695         "colorMode": "value",
1696         "graphMode": "area",
1697         "justifyMode": "auto",
1698         "orientation": "horizontal",
1699         "reduceOptions": {
1700           "calcs": [
1701             "lastNotNull"
1702           ],
1703           "fields": "",
1704           "values": false
1705         },
1706         "text": {},
1707         "textMode": "auto"
1708       },
1709       "pluginVersion": "10.0.3",
1710       "targets": [
1711         {
1712           "datasource": {
1713             "type": "prometheus",
1714             "uid": "PBFA97CFB590B2093"
1715           },
1716           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1717           "interval": "",
1718           "legendFormat": "",
1719           "refId": "A"
1720         }
1721       ],
1722       "title": "Produce Request Per Sec",
1723       "type": "stat"
1724     },
1725     {
1726       "datasource": {
1727         "type": "prometheus",
1728         "uid": "PBFA97CFB590B2093"
1729       },
1730       "description": "Metadata request rate.",
1731       "fieldConfig": {
1732         "defaults": {
1733           "mappings": [
1734             {
1735               "options": {
1736                 "match": "null",
1737                 "result": {
1738                   "text": "N/A"
1739                 }
1740               },
1741               "type": "special"
1742             }
1743           ],
1744           "thresholds": {
1745             "mode": "absolute",
1746             "steps": [
1747               {
1748                 "color": "green",
1749                 "value": null
1750               }
1751             ]
1752           },
1753           "unit": "none"
1754         },
1755         "overrides": []
1756       },
1757       "gridPos": {
1758         "h": 4,
1759         "w": 4,
1760         "x": 12,
1761         "y": 23
1762       },
1763       "id": 36,
1764       "links": [],
1765       "maxDataPoints": 100,
1766       "options": {
1767         "colorMode": "value",
1768         "graphMode": "area",
1769         "justifyMode": "auto",
1770         "orientation": "horizontal",
1771         "reduceOptions": {
1772           "calcs": [
1773             "lastNotNull"
1774           ],
1775           "fields": "",
1776           "values": false
1777         },
1778         "text": {},
1779         "textMode": "auto"
1780       },
1781       "pluginVersion": "10.0.3",
1782       "targets": [
1783         {
1784           "datasource": {
1785             "type": "prometheus",
1786             "uid": "PBFA97CFB590B2093"
1787           },
1788           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1789           "interval": "",
1790           "legendFormat": "",
1791           "refId": "A"
1792         }
1793       ],
1794       "title": "Metadata Request Per Sec",
1795       "type": "stat"
1796     },
1797     {
1798       "datasource": {
1799         "type": "prometheus",
1800         "uid": "PBFA97CFB590B2093"
1801       },
1802       "description": "Offset Commit request rate.",
1803       "fieldConfig": {
1804         "defaults": {
1805           "mappings": [
1806             {
1807               "options": {
1808                 "match": "null",
1809                 "result": {
1810                   "text": "N/A"
1811                 }
1812               },
1813               "type": "special"
1814             }
1815           ],
1816           "thresholds": {
1817             "mode": "absolute",
1818             "steps": [
1819               {
1820                 "color": "green",
1821                 "value": null
1822               }
1823             ]
1824           },
1825           "unit": "none"
1826         },
1827         "overrides": []
1828       },
1829       "gridPos": {
1830         "h": 4,
1831         "w": 4,
1832         "x": 16,
1833         "y": 23
1834       },
1835       "id": 38,
1836       "links": [],
1837       "maxDataPoints": 100,
1838       "options": {
1839         "colorMode": "value",
1840         "graphMode": "area",
1841         "justifyMode": "auto",
1842         "orientation": "horizontal",
1843         "reduceOptions": {
1844           "calcs": [
1845             "lastNotNull"
1846           ],
1847           "fields": "",
1848           "values": false
1849         },
1850         "text": {},
1851         "textMode": "auto"
1852       },
1853       "pluginVersion": "10.0.3",
1854       "targets": [
1855         {
1856           "datasource": {
1857             "type": "prometheus",
1858             "uid": "PBFA97CFB590B2093"
1859           },
1860           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1861           "interval": "",
1862           "legendFormat": "",
1863           "refId": "A"
1864         }
1865       ],
1866       "title": "Offset Commit Request Per Sec",
1867       "type": "stat"
1868     },
1869     {
1870       "datasource": {
1871         "type": "prometheus",
1872         "uid": "PBFA97CFB590B2093"
1873       },
1874       "description": "Fetch request rate.",
1875       "fieldConfig": {
1876         "defaults": {
1877           "mappings": [
1878             {
1879               "options": {
1880                 "match": "null",
1881                 "result": {
1882                   "text": "N/A"
1883                 }
1884               },
1885               "type": "special"
1886             }
1887           ],
1888           "thresholds": {
1889             "mode": "absolute",
1890             "steps": [
1891               {
1892                 "color": "green",
1893                 "value": null
1894               }
1895             ]
1896           },
1897           "unit": "none"
1898         },
1899         "overrides": []
1900       },
1901       "gridPos": {
1902         "h": 4,
1903         "w": 4,
1904         "x": 20,
1905         "y": 23
1906       },
1907       "id": 94,
1908       "links": [],
1909       "maxDataPoints": 100,
1910       "options": {
1911         "colorMode": "value",
1912         "graphMode": "area",
1913         "justifyMode": "auto",
1914         "orientation": "horizontal",
1915         "reduceOptions": {
1916           "calcs": [
1917             "lastNotNull"
1918           ],
1919           "fields": "",
1920           "values": false
1921         },
1922         "text": {},
1923         "textMode": "auto"
1924       },
1925       "pluginVersion": "10.0.3",
1926       "targets": [
1927         {
1928           "datasource": {
1929             "type": "prometheus",
1930             "uid": "PBFA97CFB590B2093"
1931           },
1932           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1933           "interval": "",
1934           "legendFormat": "",
1935           "refId": "A"
1936         }
1937       ],
1938       "title": "Broker Fetch Request Per Sec",
1939       "type": "stat"
1940     },
1941     {
1942       "collapsed": false,
1943       "datasource": {
1944         "type": "datasource",
1945         "uid": "grafana"
1946       },
1947       "gridPos": {
1948         "h": 1,
1949         "w": 24,
1950         "x": 0,
1951         "y": 27
1952       },
1953       "id": 29,
1954       "panels": [],
1955       "targets": [
1956         {
1957           "datasource": {
1958             "type": "datasource",
1959             "uid": "grafana"
1960           },
1961           "refId": "A"
1962         }
1963       ],
1964       "title": "Throughput In/Out",
1965       "type": "row"
1966     },
1967     {
1968       "datasource": {
1969         "type": "prometheus",
1970         "uid": "PBFA97CFB590B2093"
1971       },
1972       "fieldConfig": {
1973         "defaults": {
1974           "color": {
1975             "mode": "palette-classic"
1976           },
1977           "custom": {
1978             "axisCenteredZero": false,
1979             "axisColorMode": "text",
1980             "axisLabel": "Messages/s",
1981             "axisPlacement": "auto",
1982             "barAlignment": 0,
1983             "drawStyle": "line",
1984             "fillOpacity": 10,
1985             "gradientMode": "none",
1986             "hideFrom": {
1987               "legend": false,
1988               "tooltip": false,
1989               "viz": false
1990             },
1991             "lineInterpolation": "linear",
1992             "lineWidth": 2,
1993             "pointSize": 5,
1994             "scaleDistribution": {
1995               "type": "linear"
1996             },
1997             "showPoints": "never",
1998             "spanNulls": false,
1999             "stacking": {
2000               "group": "A",
2001               "mode": "none"
2002             },
2003             "thresholdsStyle": {
2004               "mode": "off"
2005             }
2006           },
2007           "links": [],
2008           "mappings": [],
2009           "min": 0,
2010           "thresholds": {
2011             "mode": "absolute",
2012             "steps": [
2013               {
2014                 "color": "green",
2015                 "value": null
2016               }
2017             ]
2018           },
2019           "unit": "cps"
2020         },
2021         "overrides": []
2022       },
2023       "gridPos": {
2024         "h": 8,
2025         "w": 8,
2026         "x": 0,
2027         "y": 28
2028       },
2029       "id": 4,
2030       "links": [],
2031       "options": {
2032         "legend": {
2033           "calcs": [
2034             "mean",
2035             "lastNotNull",
2036             "max"
2037           ],
2038           "displayMode": "table",
2039           "placement": "bottom",
2040           "showLegend": true
2041         },
2042         "tooltip": {
2043           "mode": "single",
2044           "sort": "none"
2045         }
2046       },
2047       "pluginVersion": "8.1.3",
2048       "targets": [
2049         {
2050           "datasource": {
2051             "type": "prometheus",
2052             "uid": "PBFA97CFB590B2093"
2053           },
2054           "exemplar": true,
2055           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2056           "interval": "",
2057           "intervalFactor": 2,
2058           "legendFormat": "bytes/sec",
2059           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2060           "refId": "A",
2061           "step": 4
2062         }
2063       ],
2064       "title": "Messages In",
2065       "type": "timeseries"
2066     },
2067     {
2068       "datasource": {
2069         "type": "prometheus",
2070         "uid": "PBFA97CFB590B2093"
2071       },
2072       "fieldConfig": {
2073         "defaults": {
2074           "color": {
2075             "mode": "palette-classic"
2076           },
2077           "custom": {
2078             "axisCenteredZero": false,
2079             "axisColorMode": "text",
2080             "axisLabel": "Bytes/s",
2081             "axisPlacement": "auto",
2082             "barAlignment": 0,
2083             "drawStyle": "line",
2084             "fillOpacity": 10,
2085             "gradientMode": "none",
2086             "hideFrom": {
2087               "legend": false,
2088               "tooltip": false,
2089               "viz": false
2090             },
2091             "lineInterpolation": "linear",
2092             "lineWidth": 2,
2093             "pointSize": 5,
2094             "scaleDistribution": {
2095               "type": "linear"
2096             },
2097             "showPoints": "never",
2098             "spanNulls": false,
2099             "stacking": {
2100               "group": "A",
2101               "mode": "none"
2102             },
2103             "thresholdsStyle": {
2104               "mode": "off"
2105             }
2106           },
2107           "links": [],
2108           "mappings": [],
2109           "min": 0,
2110           "thresholds": {
2111             "mode": "absolute",
2112             "steps": [
2113               {
2114                 "color": "green",
2115                 "value": null
2116               },
2117               {
2118                 "color": "red",
2119                 "value": 80
2120               }
2121             ]
2122           },
2123           "unit": "Bps"
2124         },
2125         "overrides": []
2126       },
2127       "gridPos": {
2128         "h": 8,
2129         "w": 8,
2130         "x": 8,
2131         "y": 28
2132       },
2133       "id": 5,
2134       "links": [],
2135       "options": {
2136         "legend": {
2137           "calcs": [
2138             "mean",
2139             "lastNotNull",
2140             "max"
2141           ],
2142           "displayMode": "table",
2143           "placement": "bottom",
2144           "showLegend": true
2145         },
2146         "tooltip": {
2147           "mode": "single",
2148           "sort": "none"
2149         }
2150       },
2151       "pluginVersion": "8.1.3",
2152       "targets": [
2153         {
2154           "datasource": {
2155             "type": "prometheus",
2156             "uid": "PBFA97CFB590B2093"
2157           },
2158           "exemplar": true,
2159           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2160           "format": "time_series",
2161           "interval": "",
2162           "intervalFactor": 2,
2163           "legendFormat": "bytes/sec",
2164           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2165           "refId": "A",
2166           "step": 4
2167         }
2168       ],
2169       "title": "Bytes In",
2170       "type": "timeseries"
2171     },
2172     {
2173       "datasource": {
2174         "type": "prometheus",
2175         "uid": "PBFA97CFB590B2093"
2176       },
2177       "fieldConfig": {
2178         "defaults": {
2179           "color": {
2180             "mode": "palette-classic"
2181           },
2182           "custom": {
2183             "axisCenteredZero": false,
2184             "axisColorMode": "text",
2185             "axisLabel": "Bytes/s",
2186             "axisPlacement": "auto",
2187             "barAlignment": 0,
2188             "drawStyle": "line",
2189             "fillOpacity": 10,
2190             "gradientMode": "none",
2191             "hideFrom": {
2192               "legend": false,
2193               "tooltip": false,
2194               "viz": false
2195             },
2196             "lineInterpolation": "linear",
2197             "lineWidth": 2,
2198             "pointSize": 5,
2199             "scaleDistribution": {
2200               "type": "linear"
2201             },
2202             "showPoints": "never",
2203             "spanNulls": false,
2204             "stacking": {
2205               "group": "A",
2206               "mode": "none"
2207             },
2208             "thresholdsStyle": {
2209               "mode": "off"
2210             }
2211           },
2212           "links": [],
2213           "mappings": [],
2214           "min": 0,
2215           "thresholds": {
2216             "mode": "absolute",
2217             "steps": [
2218               {
2219                 "color": "green",
2220                 "value": null
2221               },
2222               {
2223                 "color": "red",
2224                 "value": 80
2225               }
2226             ]
2227           },
2228           "unit": "Bps"
2229         },
2230         "overrides": []
2231       },
2232       "gridPos": {
2233         "h": 8,
2234         "w": 8,
2235         "x": 16,
2236         "y": 28
2237       },
2238       "id": 6,
2239       "links": [],
2240       "options": {
2241         "legend": {
2242           "calcs": [
2243             "mean",
2244             "lastNotNull",
2245             "max"
2246           ],
2247           "displayMode": "table",
2248           "placement": "bottom",
2249           "showLegend": true
2250         },
2251         "tooltip": {
2252           "mode": "single",
2253           "sort": "none"
2254         }
2255       },
2256       "pluginVersion": "8.1.3",
2257       "targets": [
2258         {
2259           "datasource": {
2260             "type": "prometheus",
2261             "uid": "PBFA97CFB590B2093"
2262           },
2263           "exemplar": true,
2264           "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2265           "interval": "",
2266           "intervalFactor": 2,
2267           "legendFormat": "bytes/sec",
2268           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2269           "refId": "A",
2270           "step": 4
2271         }
2272       ],
2273       "title": "Bytes Out",
2274       "type": "timeseries"
2275     },
2276     {
2277       "datasource": {
2278         "type": "prometheus",
2279         "uid": "PBFA97CFB590B2093"
2280       },
2281       "fieldConfig": {
2282         "defaults": {
2283           "color": {
2284             "mode": "palette-classic"
2285           },
2286           "custom": {
2287             "axisCenteredZero": false,
2288             "axisColorMode": "text",
2289             "axisLabel": "Messages/s",
2290             "axisPlacement": "auto",
2291             "barAlignment": 0,
2292             "drawStyle": "line",
2293             "fillOpacity": 10,
2294             "gradientMode": "none",
2295             "hideFrom": {
2296               "legend": false,
2297               "tooltip": false,
2298               "viz": false
2299             },
2300             "lineInterpolation": "linear",
2301             "lineWidth": 2,
2302             "pointSize": 5,
2303             "scaleDistribution": {
2304               "type": "linear"
2305             },
2306             "showPoints": "never",
2307             "spanNulls": false,
2308             "stacking": {
2309               "group": "A",
2310               "mode": "none"
2311             },
2312             "thresholdsStyle": {
2313               "mode": "off"
2314             }
2315           },
2316           "links": [],
2317           "mappings": [],
2318           "thresholds": {
2319             "mode": "absolute",
2320             "steps": [
2321               {
2322                 "color": "green",
2323                 "value": null
2324               },
2325               {
2326                 "color": "red",
2327                 "value": 80
2328               }
2329             ]
2330           },
2331           "unit": "cps"
2332         },
2333         "overrides": []
2334       },
2335       "gridPos": {
2336         "h": 8,
2337         "w": 8,
2338         "x": 0,
2339         "y": 36
2340       },
2341       "id": 10,
2342       "links": [],
2343       "options": {
2344         "legend": {
2345           "calcs": [
2346             "mean",
2347             "lastNotNull",
2348             "max"
2349           ],
2350           "displayMode": "table",
2351           "placement": "bottom",
2352           "showLegend": true
2353         },
2354         "tooltip": {
2355           "mode": "single",
2356           "sort": "none"
2357         }
2358       },
2359       "pluginVersion": "8.1.3",
2360       "targets": [
2361         {
2362           "datasource": {
2363             "type": "prometheus",
2364             "uid": "PBFA97CFB590B2093"
2365           },
2366           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2367           "format": "time_series",
2368           "interval": "",
2369           "intervalFactor": 2,
2370           "legendFormat": "{{instance}}",
2371           "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2372           "refId": "A",
2373           "step": 4
2374         }
2375       ],
2376       "title": "Messages In Per Broker",
2377       "type": "timeseries"
2378     },
2379     {
2380       "datasource": {
2381         "type": "prometheus",
2382         "uid": "PBFA97CFB590B2093"
2383       },
2384       "fieldConfig": {
2385         "defaults": {
2386           "color": {
2387             "mode": "palette-classic"
2388           },
2389           "custom": {
2390             "axisCenteredZero": false,
2391             "axisColorMode": "text",
2392             "axisLabel": "Bytes/s",
2393             "axisPlacement": "auto",
2394             "barAlignment": 0,
2395             "drawStyle": "line",
2396             "fillOpacity": 10,
2397             "gradientMode": "none",
2398             "hideFrom": {
2399               "legend": false,
2400               "tooltip": false,
2401               "viz": false
2402             },
2403             "lineInterpolation": "linear",
2404             "lineWidth": 2,
2405             "pointSize": 5,
2406             "scaleDistribution": {
2407               "type": "linear"
2408             },
2409             "showPoints": "never",
2410             "spanNulls": false,
2411             "stacking": {
2412               "group": "A",
2413               "mode": "none"
2414             },
2415             "thresholdsStyle": {
2416               "mode": "off"
2417             }
2418           },
2419           "links": [],
2420           "mappings": [],
2421           "thresholds": {
2422             "mode": "absolute",
2423             "steps": [
2424               {
2425                 "color": "green",
2426                 "value": null
2427               },
2428               {
2429                 "color": "red",
2430                 "value": 80
2431               }
2432             ]
2433           },
2434           "unit": "Bps"
2435         },
2436         "overrides": []
2437       },
2438       "gridPos": {
2439         "h": 8,
2440         "w": 8,
2441         "x": 8,
2442         "y": 36
2443       },
2444       "id": 7,
2445       "links": [],
2446       "options": {
2447         "legend": {
2448           "calcs": [
2449             "mean",
2450             "lastNotNull",
2451             "max"
2452           ],
2453           "displayMode": "table",
2454           "placement": "bottom",
2455           "showLegend": true
2456         },
2457         "tooltip": {
2458           "mode": "single",
2459           "sort": "none"
2460         }
2461       },
2462       "pluginVersion": "8.1.3",
2463       "targets": [
2464         {
2465           "datasource": {
2466             "type": "prometheus",
2467             "uid": "PBFA97CFB590B2093"
2468           },
2469           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2470           "format": "time_series",
2471           "interval": "",
2472           "intervalFactor": 2,
2473           "legendFormat": "{{instance}}",
2474           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2475           "refId": "A",
2476           "step": 4
2477         }
2478       ],
2479       "title": "Bytes In Per Broker",
2480       "type": "timeseries"
2481     },
2482     {
2483       "datasource": {
2484         "type": "prometheus",
2485         "uid": "PBFA97CFB590B2093"
2486       },
2487       "fieldConfig": {
2488         "defaults": {
2489           "color": {
2490             "mode": "palette-classic"
2491           },
2492           "custom": {
2493             "axisCenteredZero": false,
2494             "axisColorMode": "text",
2495             "axisLabel": "",
2496             "axisPlacement": "auto",
2497             "barAlignment": 0,
2498             "drawStyle": "line",
2499             "fillOpacity": 10,
2500             "gradientMode": "none",
2501             "hideFrom": {
2502               "legend": false,
2503               "tooltip": false,
2504               "viz": false
2505             },
2506             "lineInterpolation": "linear",
2507             "lineWidth": 1,
2508             "pointSize": 5,
2509             "scaleDistribution": {
2510               "type": "linear"
2511             },
2512             "showPoints": "never",
2513             "spanNulls": true,
2514             "stacking": {
2515               "group": "A",
2516               "mode": "none"
2517             },
2518             "thresholdsStyle": {
2519               "mode": "off"
2520             }
2521           },
2522           "links": [],
2523           "mappings": [],
2524           "min": 0,
2525           "thresholds": {
2526             "mode": "absolute",
2527             "steps": [
2528               {
2529                 "color": "green",
2530                 "value": null
2531               },
2532               {
2533                 "color": "red",
2534                 "value": 80
2535               }
2536             ]
2537           },
2538           "unit": "binBps"
2539         },
2540         "overrides": []
2541       },
2542       "gridPos": {
2543         "h": 8,
2544         "w": 8,
2545         "x": 16,
2546         "y": 36
2547       },
2548       "id": 9,
2549       "links": [],
2550       "options": {
2551         "legend": {
2552           "calcs": [
2553             "mean",
2554             "lastNotNull",
2555             "max"
2556           ],
2557           "displayMode": "table",
2558           "placement": "bottom",
2559           "showLegend": true
2560         },
2561         "tooltip": {
2562           "mode": "single",
2563           "sort": "none"
2564         }
2565       },
2566       "pluginVersion": "8.1.3",
2567       "targets": [
2568         {
2569           "datasource": {
2570             "type": "prometheus",
2571             "uid": "PBFA97CFB590B2093"
2572           },
2573           "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2574           "format": "time_series",
2575           "interval": "",
2576           "intervalFactor": 1,
2577           "legendFormat": "{{instance}}",
2578           "refId": "A"
2579         }
2580       ],
2581       "title": "Bytes Out Per Broker",
2582       "type": "timeseries"
2583     },
2584     {
2585       "collapsed": false,
2586       "datasource": {
2587         "type": "datasource",
2588         "uid": "grafana"
2589       },
2590       "gridPos": {
2591         "h": 1,
2592         "w": 24,
2593         "x": 0,
2594         "y": 44
2595       },
2596       "id": 44,
2597       "panels": [],
2598       "targets": [
2599         {
2600           "datasource": {
2601             "type": "datasource",
2602             "uid": "grafana"
2603           },
2604           "refId": "A"
2605         }
2606       ],
2607       "title": "Thread utilization",
2608       "type": "row"
2609     },
2610     {
2611       "datasource": {
2612         "type": "prometheus",
2613         "uid": "PBFA97CFB590B2093"
2614       },
2615       "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2616       "fieldConfig": {
2617         "defaults": {
2618           "color": {
2619             "mode": "palette-classic"
2620           },
2621           "custom": {
2622             "axisCenteredZero": false,
2623             "axisColorMode": "text",
2624             "axisLabel": "",
2625             "axisPlacement": "auto",
2626             "barAlignment": 0,
2627             "drawStyle": "line",
2628             "fillOpacity": 10,
2629             "gradientMode": "none",
2630             "hideFrom": {
2631               "legend": false,
2632               "tooltip": false,
2633               "viz": false
2634             },
2635             "lineInterpolation": "linear",
2636             "lineWidth": 1,
2637             "pointSize": 5,
2638             "scaleDistribution": {
2639               "type": "linear"
2640             },
2641             "showPoints": "never",
2642             "spanNulls": true,
2643             "stacking": {
2644               "group": "A",
2645               "mode": "none"
2646             },
2647             "thresholdsStyle": {
2648               "mode": "off"
2649             }
2650           },
2651           "links": [],
2652           "mappings": [],
2653           "thresholds": {
2654             "mode": "absolute",
2655             "steps": [
2656               {
2657                 "color": "green",
2658                 "value": null
2659               },
2660               {
2661                 "color": "red",
2662                 "value": 80
2663               }
2664             ]
2665           },
2666           "unit": "percentunit"
2667         },
2668         "overrides": []
2669       },
2670       "gridPos": {
2671         "h": 12,
2672         "w": 12,
2673         "x": 0,
2674         "y": 45
2675       },
2676       "id": 24,
2677       "options": {
2678         "legend": {
2679           "calcs": [
2680             "mean",
2681             "lastNotNull",
2682             "max"
2683           ],
2684           "displayMode": "table",
2685           "placement": "bottom",
2686           "showLegend": true
2687         },
2688         "tooltip": {
2689           "mode": "single",
2690           "sort": "none"
2691         }
2692       },
2693       "pluginVersion": "8.1.3",
2694       "targets": [
2695         {
2696           "datasource": {
2697             "type": "prometheus",
2698             "uid": "PBFA97CFB590B2093"
2699           },
2700           "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2701           "interval": "",
2702           "legendFormat": "{{instance}}",
2703           "refId": "A"
2704         }
2705       ],
2706       "title": "Network Processor Avg Usage Percent",
2707       "type": "timeseries"
2708     },
2709     {
2710       "datasource": {
2711         "type": "prometheus",
2712         "uid": "PBFA97CFB590B2093"
2713       },
2714       "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2715       "fieldConfig": {
2716         "defaults": {
2717           "color": {
2718             "mode": "palette-classic"
2719           },
2720           "custom": {
2721             "axisCenteredZero": false,
2722             "axisColorMode": "text",
2723             "axisLabel": "",
2724             "axisPlacement": "auto",
2725             "barAlignment": 0,
2726             "drawStyle": "line",
2727             "fillOpacity": 10,
2728             "gradientMode": "none",
2729             "hideFrom": {
2730               "legend": false,
2731               "tooltip": false,
2732               "viz": false
2733             },
2734             "lineInterpolation": "linear",
2735             "lineWidth": 1,
2736             "pointSize": 5,
2737             "scaleDistribution": {
2738               "type": "linear"
2739             },
2740             "showPoints": "never",
2741             "spanNulls": true,
2742             "stacking": {
2743               "group": "A",
2744               "mode": "none"
2745             },
2746             "thresholdsStyle": {
2747               "mode": "off"
2748             }
2749           },
2750           "links": [],
2751           "mappings": [],
2752           "min": 0,
2753           "thresholds": {
2754             "mode": "absolute",
2755             "steps": [
2756               {
2757                 "color": "green",
2758                 "value": null
2759               },
2760               {
2761                 "color": "red",
2762                 "value": 80
2763               }
2764             ]
2765           },
2766           "unit": "percentunit"
2767         },
2768         "overrides": []
2769       },
2770       "gridPos": {
2771         "h": 12,
2772         "w": 12,
2773         "x": 12,
2774         "y": 45
2775       },
2776       "id": 25,
2777       "options": {
2778         "legend": {
2779           "calcs": [
2780             "mean",
2781             "lastNotNull",
2782             "max"
2783           ],
2784           "displayMode": "table",
2785           "placement": "bottom",
2786           "showLegend": true
2787         },
2788         "tooltip": {
2789           "mode": "single",
2790           "sort": "none"
2791         }
2792       },
2793       "pluginVersion": "8.1.3",
2794       "targets": [
2795         {
2796           "datasource": {
2797             "type": "prometheus",
2798             "uid": "PBFA97CFB590B2093"
2799           },
2800           "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2801           "interval": "",
2802           "legendFormat": "{{instance}}",
2803           "refId": "A"
2804         }
2805       ],
2806       "title": "Request Handler Avg Percent",
2807       "type": "timeseries"
2808     },
2809     {
2810       "collapsed": false,
2811       "datasource": {
2812         "type": "datasource",
2813         "uid": "grafana"
2814       },
2815       "gridPos": {
2816         "h": 1,
2817         "w": 24,
2818         "x": 0,
2819         "y": 57
2820       },
2821       "id": 82,
2822       "panels": [],
2823       "targets": [
2824         {
2825           "datasource": {
2826             "type": "datasource",
2827             "uid": "grafana"
2828           },
2829           "refId": "A"
2830         }
2831       ],
2832       "title": "Isr Shrinks / Expands",
2833       "type": "row"
2834     },
2835     {
2836       "datasource": {
2837         "type": "prometheus",
2838         "uid": "PBFA97CFB590B2093"
2839       },
2840       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2841       "fieldConfig": {
2842         "defaults": {
2843           "color": {
2844             "mode": "palette-classic"
2845           },
2846           "custom": {
2847             "axisCenteredZero": false,
2848             "axisColorMode": "text",
2849             "axisLabel": "",
2850             "axisPlacement": "auto",
2851             "barAlignment": 0,
2852             "drawStyle": "line",
2853             "fillOpacity": 10,
2854             "gradientMode": "none",
2855             "hideFrom": {
2856               "legend": false,
2857               "tooltip": false,
2858               "viz": false
2859             },
2860             "lineInterpolation": "linear",
2861             "lineWidth": 1,
2862             "pointSize": 5,
2863             "scaleDistribution": {
2864               "type": "linear"
2865             },
2866             "showPoints": "never",
2867             "spanNulls": true,
2868             "stacking": {
2869               "group": "A",
2870               "mode": "none"
2871             },
2872             "thresholdsStyle": {
2873               "mode": "off"
2874             }
2875           },
2876           "links": [],
2877           "mappings": [],
2878           "min": 0,
2879           "thresholds": {
2880             "mode": "absolute",
2881             "steps": [
2882               {
2883                 "color": "green",
2884                 "value": null
2885               },
2886               {
2887                 "color": "red",
2888                 "value": 80
2889               }
2890             ]
2891           },
2892           "unit": "ops"
2893         },
2894         "overrides": [
2895           {
2896             "__systemRef": "hideSeriesFrom",
2897             "matcher": {
2898               "id": "byNames",
2899               "options": {
2900                 "mode": "exclude",
2901                 "names": [
2902                   "kafka-2:80"
2903                 ],
2904                 "prefix": "All except:",
2905                 "readOnly": true
2906               }
2907             },
2908             "properties": [
2909               {
2910                 "id": "custom.hideFrom",
2911                 "value": {
2912                   "legend": false,
2913                   "tooltip": false,
2914                   "viz": true
2915                 }
2916               }
2917             ]
2918           }
2919         ]
2920       },
2921       "gridPos": {
2922         "h": 9,
2923         "w": 12,
2924         "x": 0,
2925         "y": 58
2926       },
2927       "id": 80,
2928       "options": {
2929         "legend": {
2930           "calcs": [
2931             "mean",
2932             "lastNotNull",
2933             "max"
2934           ],
2935           "displayMode": "table",
2936           "placement": "bottom",
2937           "showLegend": true
2938         },
2939         "tooltip": {
2940           "mode": "single",
2941           "sort": "none"
2942         }
2943       },
2944       "pluginVersion": "8.1.3",
2945       "targets": [
2946         {
2947           "datasource": {
2948             "type": "prometheus",
2949             "uid": "PBFA97CFB590B2093"
2950           },
2951           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
2952           "interval": "",
2953           "legendFormat": "{{instance}}",
2954           "refId": "A"
2955         }
2956       ],
2957       "title": "IsrShrinks per Sec",
2958       "type": "timeseries"
2959     },
2960     {
2961       "datasource": {
2962         "type": "prometheus",
2963         "uid": "PBFA97CFB590B2093"
2964       },
2965       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
2966       "fieldConfig": {
2967         "defaults": {
2968           "color": {
2969             "mode": "palette-classic"
2970           },
2971           "custom": {
2972             "axisCenteredZero": false,
2973             "axisColorMode": "text",
2974             "axisLabel": "",
2975             "axisPlacement": "auto",
2976             "barAlignment": 0,
2977             "drawStyle": "line",
2978             "fillOpacity": 10,
2979             "gradientMode": "none",
2980             "hideFrom": {
2981               "legend": false,
2982               "tooltip": false,
2983               "viz": false
2984             },
2985             "lineInterpolation": "linear",
2986             "lineWidth": 1,
2987             "pointSize": 5,
2988             "scaleDistribution": {
2989               "type": "linear"
2990             },
2991             "showPoints": "never",
2992             "spanNulls": true,
2993             "stacking": {
2994               "group": "A",
2995               "mode": "none"
2996             },
2997             "thresholdsStyle": {
2998               "mode": "off"
2999             }
3000           },
3001           "links": [],
3002           "mappings": [],
3003           "min": 0,
3004           "thresholds": {
3005             "mode": "absolute",
3006             "steps": [
3007               {
3008                 "color": "green",
3009                 "value": null
3010               },
3011               {
3012                 "color": "red",
3013                 "value": 80
3014               }
3015             ]
3016           },
3017           "unit": "ops"
3018         },
3019         "overrides": [
3020           {
3021             "__systemRef": "hideSeriesFrom",
3022             "matcher": {
3023               "id": "byNames",
3024               "options": {
3025                 "mode": "exclude",
3026                 "names": [
3027                   "kafka-2:80"
3028                 ],
3029                 "prefix": "All except:",
3030                 "readOnly": true
3031               }
3032             },
3033             "properties": [
3034               {
3035                 "id": "custom.hideFrom",
3036                 "value": {
3037                   "legend": false,
3038                   "tooltip": false,
3039                   "viz": true
3040                 }
3041               }
3042             ]
3043           }
3044         ]
3045       },
3046       "gridPos": {
3047         "h": 9,
3048         "w": 12,
3049         "x": 12,
3050         "y": 58
3051       },
3052       "id": 83,
3053       "options": {
3054         "legend": {
3055           "calcs": [
3056             "mean",
3057             "lastNotNull",
3058             "max"
3059           ],
3060           "displayMode": "table",
3061           "placement": "bottom",
3062           "showLegend": true
3063         },
3064         "tooltip": {
3065           "mode": "single",
3066           "sort": "none"
3067         }
3068       },
3069       "pluginVersion": "8.1.3",
3070       "targets": [
3071         {
3072           "datasource": {
3073             "type": "prometheus",
3074             "uid": "PBFA97CFB590B2093"
3075           },
3076           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3077           "hide": false,
3078           "interval": "",
3079           "legendFormat": "{{instance}}",
3080           "refId": "A"
3081         }
3082       ],
3083       "title": "IsrExpands per Sec",
3084       "type": "timeseries"
3085     },
3086     {
3087       "collapsed": false,
3088       "datasource": {
3089         "type": "datasource",
3090         "uid": "grafana"
3091       },
3092       "gridPos": {
3093         "h": 1,
3094         "w": 24,
3095         "x": 0,
3096         "y": 67
3097       },
3098       "id": 53,
3099       "panels": [],
3100       "targets": [
3101         {
3102           "datasource": {
3103             "type": "datasource",
3104             "uid": "grafana"
3105           },
3106           "refId": "A"
3107         }
3108       ],
3109       "title": "Logs size",
3110       "type": "row"
3111     },
3112     {
3113       "datasource": {
3114         "type": "prometheus",
3115         "uid": "PBFA97CFB590B2093"
3116       },
3117       "fieldConfig": {
3118         "defaults": {
3119           "color": {
3120             "mode": "palette-classic"
3121           },
3122           "custom": {
3123             "axisCenteredZero": false,
3124             "axisColorMode": "text",
3125             "axisLabel": "",
3126             "axisPlacement": "auto",
3127             "barAlignment": 0,
3128             "drawStyle": "line",
3129             "fillOpacity": 10,
3130             "gradientMode": "none",
3131             "hideFrom": {
3132               "legend": false,
3133               "tooltip": false,
3134               "viz": false
3135             },
3136             "lineInterpolation": "linear",
3137             "lineWidth": 1,
3138             "pointSize": 5,
3139             "scaleDistribution": {
3140               "type": "linear"
3141             },
3142             "showPoints": "never",
3143             "spanNulls": true,
3144             "stacking": {
3145               "group": "A",
3146               "mode": "none"
3147             },
3148             "thresholdsStyle": {
3149               "mode": "off"
3150             }
3151           },
3152           "links": [],
3153           "mappings": [],
3154           "thresholds": {
3155             "mode": "absolute",
3156             "steps": [
3157               {
3158                 "color": "green",
3159                 "value": null
3160               },
3161               {
3162                 "color": "red",
3163                 "value": 80
3164               }
3165             ]
3166           },
3167           "unit": "bytes"
3168         },
3169         "overrides": []
3170       },
3171       "gridPos": {
3172         "h": 9,
3173         "w": 12,
3174         "x": 0,
3175         "y": 68
3176       },
3177       "id": 55,
3178       "options": {
3179         "legend": {
3180           "calcs": [
3181             "mean",
3182             "lastNotNull",
3183             "max"
3184           ],
3185           "displayMode": "table",
3186           "placement": "bottom",
3187           "showLegend": true
3188         },
3189         "tooltip": {
3190           "mode": "single",
3191           "sort": "none"
3192         }
3193       },
3194       "pluginVersion": "8.1.3",
3195       "targets": [
3196         {
3197           "datasource": {
3198             "type": "prometheus",
3199             "uid": "PBFA97CFB590B2093"
3200           },
3201           "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3202           "legendFormat": "{{topic}}",
3203           "refId": "A"
3204         }
3205       ],
3206       "title": "Log size per Topic",
3207       "type": "timeseries"
3208     },
3209     {
3210       "datasource": {
3211         "type": "prometheus",
3212         "uid": "PBFA97CFB590B2093"
3213       },
3214       "fieldConfig": {
3215         "defaults": {
3216           "color": {
3217             "mode": "palette-classic"
3218           },
3219           "custom": {
3220             "axisCenteredZero": false,
3221             "axisColorMode": "text",
3222             "axisLabel": "",
3223             "axisPlacement": "auto",
3224             "barAlignment": 0,
3225             "drawStyle": "line",
3226             "fillOpacity": 10,
3227             "gradientMode": "none",
3228             "hideFrom": {
3229               "legend": false,
3230               "tooltip": false,
3231               "viz": false
3232             },
3233             "lineInterpolation": "linear",
3234             "lineWidth": 1,
3235             "pointSize": 5,
3236             "scaleDistribution": {
3237               "type": "linear"
3238             },
3239             "showPoints": "never",
3240             "spanNulls": true,
3241             "stacking": {
3242               "group": "A",
3243               "mode": "none"
3244             },
3245             "thresholdsStyle": {
3246               "mode": "off"
3247             }
3248           },
3249           "links": [],
3250           "mappings": [],
3251           "thresholds": {
3252             "mode": "absolute",
3253             "steps": [
3254               {
3255                 "color": "green",
3256                 "value": null
3257               },
3258               {
3259                 "color": "red",
3260                 "value": 80
3261               }
3262             ]
3263           },
3264           "unit": "bytes"
3265         },
3266         "overrides": []
3267       },
3268       "gridPos": {
3269         "h": 9,
3270         "w": 12,
3271         "x": 12,
3272         "y": 68
3273       },
3274       "id": 56,
3275       "options": {
3276         "legend": {
3277           "calcs": [
3278             "mean",
3279             "lastNotNull",
3280             "max"
3281           ],
3282           "displayMode": "table",
3283           "placement": "bottom",
3284           "showLegend": true
3285         },
3286         "tooltip": {
3287           "mode": "single",
3288           "sort": "none"
3289         }
3290       },
3291       "pluginVersion": "8.1.3",
3292       "targets": [
3293         {
3294           "datasource": {
3295             "type": "prometheus",
3296             "uid": "PBFA97CFB590B2093"
3297           },
3298           "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3299           "legendFormat": "{{instance}}",
3300           "refId": "A"
3301         }
3302       ],
3303       "title": "Log size per Broker",
3304       "type": "timeseries"
3305     },
3306     {
3307       "collapsed": false,
3308       "datasource": {
3309         "type": "datasource",
3310         "uid": "grafana"
3311       },
3312       "gridPos": {
3313         "h": 1,
3314         "w": 24,
3315         "x": 0,
3316         "y": 77
3317       },
3318       "id": 58,
3319       "panels": [],
3320       "targets": [
3321         {
3322           "datasource": {
3323             "type": "datasource",
3324             "uid": "grafana"
3325           },
3326           "refId": "A"
3327         }
3328       ],
3329       "title": "Producer Performance",
3330       "type": "row"
3331     },
3332     {
3333       "datasource": {
3334         "type": "prometheus",
3335         "uid": "PBFA97CFB590B2093"
3336       },
3337       "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3338       "fieldConfig": {
3339         "defaults": {
3340           "color": {
3341             "mode": "palette-classic"
3342           },
3343           "custom": {
3344             "axisCenteredZero": false,
3345             "axisColorMode": "text",
3346             "axisLabel": "",
3347             "axisPlacement": "auto",
3348             "barAlignment": 0,
3349             "drawStyle": "line",
3350             "fillOpacity": 10,
3351             "gradientMode": "none",
3352             "hideFrom": {
3353               "legend": false,
3354               "tooltip": false,
3355               "viz": false
3356             },
3357             "lineInterpolation": "linear",
3358             "lineWidth": 1,
3359             "pointSize": 5,
3360             "scaleDistribution": {
3361               "type": "linear"
3362             },
3363             "showPoints": "never",
3364             "spanNulls": true,
3365             "stacking": {
3366               "group": "A",
3367               "mode": "none"
3368             },
3369             "thresholdsStyle": {
3370               "mode": "off"
3371             }
3372           },
3373           "links": [],
3374           "mappings": [],
3375           "thresholds": {
3376             "mode": "absolute",
3377             "steps": [
3378               {
3379                 "color": "green",
3380                 "value": null
3381               },
3382               {
3383                 "color": "red",
3384                 "value": 80
3385               }
3386             ]
3387           },
3388           "unit": "ms"
3389         },
3390         "overrides": []
3391       },
3392       "gridPos": {
3393         "h": 9,
3394         "w": 12,
3395         "x": 0,
3396         "y": 78
3397       },
3398       "id": 60,
3399       "options": {
3400         "legend": {
3401           "calcs": [
3402             "mean",
3403             "lastNotNull",
3404             "max"
3405           ],
3406           "displayMode": "list",
3407           "placement": "bottom",
3408           "showLegend": true
3409         },
3410         "tooltip": {
3411           "mode": "single",
3412           "sort": "none"
3413         }
3414       },
3415       "pluginVersion": "8.1.3",
3416       "targets": [
3417         {
3418           "datasource": {
3419             "type": "prometheus",
3420             "uid": "PBFA97CFB590B2093"
3421           },
3422           "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3423           "hide": false,
3424           "legendFormat": "{{instance}} - {{quantile}}",
3425           "refId": "A"
3426         }
3427       ],
3428       "title": "Producer - RequestQueueTimeMs",
3429       "type": "timeseries"
3430     },
3431     {
3432       "datasource": {
3433         "type": "prometheus",
3434         "uid": "PBFA97CFB590B2093"
3435       },
3436       "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3437       "fieldConfig": {
3438         "defaults": {
3439           "color": {
3440             "mode": "palette-classic"
3441           },
3442           "custom": {
3443             "axisCenteredZero": false,
3444             "axisColorMode": "text",
3445             "axisLabel": "",
3446             "axisPlacement": "auto",
3447             "barAlignment": 0,
3448             "drawStyle": "line",
3449             "fillOpacity": 10,
3450             "gradientMode": "none",
3451             "hideFrom": {
3452               "legend": false,
3453               "tooltip": false,
3454               "viz": false
3455             },
3456             "lineInterpolation": "linear",
3457             "lineWidth": 1,
3458             "pointSize": 5,
3459             "scaleDistribution": {
3460               "type": "linear"
3461             },
3462             "showPoints": "never",
3463             "spanNulls": true,
3464             "stacking": {
3465               "group": "A",
3466               "mode": "none"
3467             },
3468             "thresholdsStyle": {
3469               "mode": "off"
3470             }
3471           },
3472           "links": [],
3473           "mappings": [],
3474           "thresholds": {
3475             "mode": "absolute",
3476             "steps": [
3477               {
3478                 "color": "green",
3479                 "value": null
3480               },
3481               {
3482                 "color": "red",
3483                 "value": 80
3484               }
3485             ]
3486           },
3487           "unit": "ms"
3488         },
3489         "overrides": []
3490       },
3491       "gridPos": {
3492         "h": 9,
3493         "w": 12,
3494         "x": 12,
3495         "y": 78
3496       },
3497       "id": 61,
3498       "options": {
3499         "legend": {
3500           "calcs": [
3501             "mean",
3502             "lastNotNull",
3503             "max"
3504           ],
3505           "displayMode": "list",
3506           "placement": "bottom",
3507           "showLegend": true
3508         },
3509         "tooltip": {
3510           "mode": "single",
3511           "sort": "none"
3512         }
3513       },
3514       "pluginVersion": "8.1.3",
3515       "targets": [
3516         {
3517           "datasource": {
3518             "type": "prometheus",
3519             "uid": "PBFA97CFB590B2093"
3520           },
3521           "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3522           "hide": false,
3523           "legendFormat": "{{instance}} - {{quantile}}",
3524           "refId": "A"
3525         }
3526       ],
3527       "title": "Producer - LocalTimeMs",
3528       "type": "timeseries"
3529     },
3530     {
3531       "datasource": {
3532         "type": "prometheus",
3533         "uid": "PBFA97CFB590B2093"
3534       },
3535       "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3536       "fieldConfig": {
3537         "defaults": {
3538           "color": {
3539             "mode": "palette-classic"
3540           },
3541           "custom": {
3542             "axisCenteredZero": false,
3543             "axisColorMode": "text",
3544             "axisLabel": "",
3545             "axisPlacement": "auto",
3546             "barAlignment": 0,
3547             "drawStyle": "line",
3548             "fillOpacity": 10,
3549             "gradientMode": "none",
3550             "hideFrom": {
3551               "legend": false,
3552               "tooltip": false,
3553               "viz": false
3554             },
3555             "lineInterpolation": "linear",
3556             "lineWidth": 1,
3557             "pointSize": 5,
3558             "scaleDistribution": {
3559               "type": "linear"
3560             },
3561             "showPoints": "never",
3562             "spanNulls": true,
3563             "stacking": {
3564               "group": "A",
3565               "mode": "none"
3566             },
3567             "thresholdsStyle": {
3568               "mode": "off"
3569             }
3570           },
3571           "links": [],
3572           "mappings": [],
3573           "thresholds": {
3574             "mode": "absolute",
3575             "steps": [
3576               {
3577                 "color": "green",
3578                 "value": null
3579               },
3580               {
3581                 "color": "red",
3582                 "value": 80
3583               }
3584             ]
3585           },
3586           "unit": "ms"
3587         },
3588         "overrides": []
3589       },
3590       "gridPos": {
3591         "h": 10,
3592         "w": 8,
3593         "x": 0,
3594         "y": 87
3595       },
3596       "id": 62,
3597       "options": {
3598         "legend": {
3599           "calcs": [
3600             "mean",
3601             "lastNotNull",
3602             "max"
3603           ],
3604           "displayMode": "table",
3605           "placement": "bottom",
3606           "showLegend": true
3607         },
3608         "tooltip": {
3609           "mode": "single",
3610           "sort": "none"
3611         }
3612       },
3613       "pluginVersion": "8.1.3",
3614       "targets": [
3615         {
3616           "datasource": {
3617             "type": "prometheus",
3618             "uid": "PBFA97CFB590B2093"
3619           },
3620           "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3621           "hide": false,
3622           "legendFormat": "{{instance}} - {{quantile}}",
3623           "refId": "A"
3624         }
3625       ],
3626       "title": "Producer - RemoteTimeMs",
3627       "type": "timeseries"
3628     },
3629     {
3630       "datasource": {
3631         "type": "prometheus",
3632         "uid": "PBFA97CFB590B2093"
3633       },
3634       "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3635       "fieldConfig": {
3636         "defaults": {
3637           "color": {
3638             "mode": "palette-classic"
3639           },
3640           "custom": {
3641             "axisCenteredZero": false,
3642             "axisColorMode": "text",
3643             "axisLabel": "",
3644             "axisPlacement": "auto",
3645             "barAlignment": 0,
3646             "drawStyle": "line",
3647             "fillOpacity": 10,
3648             "gradientMode": "none",
3649             "hideFrom": {
3650               "legend": false,
3651               "tooltip": false,
3652               "viz": false
3653             },
3654             "lineInterpolation": "linear",
3655             "lineWidth": 1,
3656             "pointSize": 5,
3657             "scaleDistribution": {
3658               "type": "linear"
3659             },
3660             "showPoints": "never",
3661             "spanNulls": true,
3662             "stacking": {
3663               "group": "A",
3664               "mode": "none"
3665             },
3666             "thresholdsStyle": {
3667               "mode": "off"
3668             }
3669           },
3670           "links": [],
3671           "mappings": [],
3672           "thresholds": {
3673             "mode": "absolute",
3674             "steps": [
3675               {
3676                 "color": "green",
3677                 "value": null
3678               },
3679               {
3680                 "color": "red",
3681                 "value": 80
3682               }
3683             ]
3684           },
3685           "unit": "ms"
3686         },
3687         "overrides": []
3688       },
3689       "gridPos": {
3690         "h": 10,
3691         "w": 8,
3692         "x": 8,
3693         "y": 87
3694       },
3695       "id": 63,
3696       "options": {
3697         "legend": {
3698           "calcs": [
3699             "mean",
3700             "lastNotNull",
3701             "max"
3702           ],
3703           "displayMode": "table",
3704           "placement": "bottom",
3705           "showLegend": true
3706         },
3707         "tooltip": {
3708           "mode": "single",
3709           "sort": "none"
3710         }
3711       },
3712       "pluginVersion": "8.1.3",
3713       "targets": [
3714         {
3715           "datasource": {
3716             "type": "prometheus",
3717             "uid": "PBFA97CFB590B2093"
3718           },
3719           "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3720           "hide": false,
3721           "legendFormat": "{{instance}} - {{quantile}}",
3722           "refId": "A"
3723         }
3724       ],
3725       "title": "Producer - ResponseQueueTimeMs",
3726       "type": "timeseries"
3727     },
3728     {
3729       "datasource": {
3730         "type": "prometheus",
3731         "uid": "PBFA97CFB590B2093"
3732       },
3733       "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
3734       "fieldConfig": {
3735         "defaults": {
3736           "color": {
3737             "mode": "palette-classic"
3738           },
3739           "custom": {
3740             "axisCenteredZero": false,
3741             "axisColorMode": "text",
3742             "axisLabel": "",
3743             "axisPlacement": "auto",
3744             "barAlignment": 0,
3745             "drawStyle": "line",
3746             "fillOpacity": 10,
3747             "gradientMode": "none",
3748             "hideFrom": {
3749               "legend": false,
3750               "tooltip": false,
3751               "viz": false
3752             },
3753             "lineInterpolation": "linear",
3754             "lineWidth": 1,
3755             "pointSize": 5,
3756             "scaleDistribution": {
3757               "type": "linear"
3758             },
3759             "showPoints": "never",
3760             "spanNulls": true,
3761             "stacking": {
3762               "group": "A",
3763               "mode": "none"
3764             },
3765             "thresholdsStyle": {
3766               "mode": "off"
3767             }
3768           },
3769           "links": [],
3770           "mappings": [],
3771           "min": 0,
3772           "thresholds": {
3773             "mode": "absolute",
3774             "steps": [
3775               {
3776                 "color": "green",
3777                 "value": null
3778               },
3779               {
3780                 "color": "red",
3781                 "value": 80
3782               }
3783             ]
3784           },
3785           "unit": "ms"
3786         },
3787         "overrides": []
3788       },
3789       "gridPos": {
3790         "h": 10,
3791         "w": 8,
3792         "x": 16,
3793         "y": 87
3794       },
3795       "id": 64,
3796       "options": {
3797         "legend": {
3798           "calcs": [
3799             "mean",
3800             "lastNotNull",
3801             "max"
3802           ],
3803           "displayMode": "table",
3804           "placement": "bottom",
3805           "showLegend": true
3806         },
3807         "tooltip": {
3808           "mode": "single",
3809           "sort": "none"
3810         }
3811       },
3812       "pluginVersion": "8.1.3",
3813       "targets": [
3814         {
3815           "datasource": {
3816             "type": "prometheus",
3817             "uid": "PBFA97CFB590B2093"
3818           },
3819           "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3820           "hide": false,
3821           "legendFormat": "{{instance}} - {{quantile}}",
3822           "refId": "A"
3823         }
3824       ],
3825       "title": "Producer - ResponseSendTimeMs",
3826       "type": "timeseries"
3827     },
3828     {
3829       "collapsed": true,
3830       "datasource": {
3831         "type": "datasource",
3832         "uid": "grafana"
3833       },
3834       "gridPos": {
3835         "h": 1,
3836         "w": 24,
3837         "x": 0,
3838         "y": 97
3839       },
3840       "id": 68,
3841       "panels": [
3842         {
3843           "datasource": {
3844             "type": "prometheus",
3845             "uid": "PBFA97CFB590B2093"
3846           },
3847           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3848           "fieldConfig": {
3849             "defaults": {
3850               "color": {
3851                 "mode": "palette-classic"
3852               },
3853               "custom": {
3854                 "axisLabel": "",
3855                 "axisPlacement": "auto",
3856                 "barAlignment": 0,
3857                 "drawStyle": "line",
3858                 "fillOpacity": 10,
3859                 "gradientMode": "none",
3860                 "hideFrom": {
3861                   "legend": false,
3862                   "tooltip": false,
3863                   "viz": false
3864                 },
3865                 "lineInterpolation": "linear",
3866                 "lineWidth": 1,
3867                 "pointSize": 5,
3868                 "scaleDistribution": {
3869                   "type": "linear"
3870                 },
3871                 "showPoints": "never",
3872                 "spanNulls": true,
3873                 "stacking": {
3874                   "group": "A",
3875                   "mode": "none"
3876                 },
3877                 "thresholdsStyle": {
3878                   "mode": "off"
3879                 }
3880               },
3881               "links": [],
3882               "mappings": [],
3883               "min": 0,
3884               "thresholds": {
3885                 "mode": "absolute",
3886                 "steps": [
3887                   {
3888                     "color": "green"
3889                   },
3890                   {
3891                     "color": "red",
3892                     "value": 80
3893                   }
3894                 ]
3895               },
3896               "unit": "ms"
3897             },
3898             "overrides": []
3899           },
3900           "gridPos": {
3901             "h": 9,
3902             "w": 12,
3903             "x": 0,
3904             "y": 30
3905           },
3906           "id": 69,
3907           "options": {
3908             "legend": {
3909               "calcs": [
3910                 "mean",
3911                 "lastNotNull",
3912                 "max"
3913               ],
3914               "displayMode": "list",
3915               "placement": "bottom",
3916               "showLegend": true
3917             },
3918             "tooltip": {
3919               "mode": "single"
3920             }
3921           },
3922           "pluginVersion": "8.1.3",
3923           "targets": [
3924             {
3925               "datasource": {
3926                 "type": "prometheus",
3927                 "uid": "PBFA97CFB590B2093"
3928               },
3929               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
3930               "hide": false,
3931               "legendFormat": "{{instance}} - {{quantile}}",
3932               "refId": "A"
3933             }
3934           ],
3935           "title": "Consumer - RequestQueueTimeMs",
3936           "type": "timeseries"
3937         },
3938         {
3939           "datasource": {
3940             "type": "prometheus",
3941             "uid": "PBFA97CFB590B2093"
3942           },
3943           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3944           "fieldConfig": {
3945             "defaults": {
3946               "color": {
3947                 "mode": "palette-classic"
3948               },
3949               "custom": {
3950                 "axisLabel": "",
3951                 "axisPlacement": "auto",
3952                 "barAlignment": 0,
3953                 "drawStyle": "line",
3954                 "fillOpacity": 10,
3955                 "gradientMode": "none",
3956                 "hideFrom": {
3957                   "legend": false,
3958                   "tooltip": false,
3959                   "viz": false
3960                 },
3961                 "lineInterpolation": "linear",
3962                 "lineWidth": 1,
3963                 "pointSize": 5,
3964                 "scaleDistribution": {
3965                   "type": "linear"
3966                 },
3967                 "showPoints": "never",
3968                 "spanNulls": true,
3969                 "stacking": {
3970                   "group": "A",
3971                   "mode": "none"
3972                 },
3973                 "thresholdsStyle": {
3974                   "mode": "off"
3975                 }
3976               },
3977               "links": [],
3978               "mappings": [],
3979               "thresholds": {
3980                 "mode": "absolute",
3981                 "steps": [
3982                   {
3983                     "color": "green"
3984                   },
3985                   {
3986                     "color": "red",
3987                     "value": 80
3988                   }
3989                 ]
3990               },
3991               "unit": "ms"
3992             },
3993             "overrides": []
3994           },
3995           "gridPos": {
3996             "h": 9,
3997             "w": 12,
3998             "x": 12,
3999             "y": 30
4000           },
4001           "id": 70,
4002           "options": {
4003             "legend": {
4004               "calcs": [
4005                 "mean",
4006                 "lastNotNull",
4007                 "max"
4008               ],
4009               "displayMode": "list",
4010               "placement": "bottom",
4011               "showLegend": true
4012             },
4013             "tooltip": {
4014               "mode": "single"
4015             }
4016           },
4017           "pluginVersion": "8.1.3",
4018           "targets": [
4019             {
4020               "datasource": {
4021                 "type": "prometheus",
4022                 "uid": "PBFA97CFB590B2093"
4023               },
4024               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4025               "hide": false,
4026               "legendFormat": "{{instance}} - {{quantile}}",
4027               "refId": "A"
4028             }
4029           ],
4030           "title": "Consumer - LocalTimeMs",
4031           "type": "timeseries"
4032         },
4033         {
4034           "datasource": {
4035             "type": "prometheus",
4036             "uid": "PBFA97CFB590B2093"
4037           },
4038           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4039           "fieldConfig": {
4040             "defaults": {
4041               "color": {
4042                 "mode": "palette-classic"
4043               },
4044               "custom": {
4045                 "axisLabel": "",
4046                 "axisPlacement": "auto",
4047                 "barAlignment": 0,
4048                 "drawStyle": "line",
4049                 "fillOpacity": 10,
4050                 "gradientMode": "none",
4051                 "hideFrom": {
4052                   "legend": false,
4053                   "tooltip": false,
4054                   "viz": false
4055                 },
4056                 "lineInterpolation": "linear",
4057                 "lineWidth": 1,
4058                 "pointSize": 5,
4059                 "scaleDistribution": {
4060                   "type": "linear"
4061                 },
4062                 "showPoints": "never",
4063                 "spanNulls": true,
4064                 "stacking": {
4065                   "group": "A",
4066                   "mode": "none"
4067                 },
4068                 "thresholdsStyle": {
4069                   "mode": "off"
4070                 }
4071               },
4072               "links": [],
4073               "mappings": [],
4074               "thresholds": {
4075                 "mode": "absolute",
4076                 "steps": [
4077                   {
4078                     "color": "green"
4079                   },
4080                   {
4081                     "color": "red",
4082                     "value": 80
4083                   }
4084                 ]
4085               },
4086               "unit": "ms"
4087             },
4088             "overrides": []
4089           },
4090           "gridPos": {
4091             "h": 9,
4092             "w": 8,
4093             "x": 0,
4094             "y": 39
4095           },
4096           "id": 71,
4097           "options": {
4098             "legend": {
4099               "calcs": [
4100                 "mean",
4101                 "lastNotNull",
4102                 "max"
4103               ],
4104               "displayMode": "table",
4105               "placement": "bottom",
4106               "showLegend": true
4107             },
4108             "tooltip": {
4109               "mode": "single"
4110             }
4111           },
4112           "pluginVersion": "8.1.3",
4113           "targets": [
4114             {
4115               "datasource": {
4116                 "type": "prometheus",
4117                 "uid": "PBFA97CFB590B2093"
4118               },
4119               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4120               "hide": false,
4121               "legendFormat": "{{instance}} - {{quantile}}",
4122               "refId": "A"
4123             }
4124           ],
4125           "title": "Consumer - RemoteTimeMs",
4126           "type": "timeseries"
4127         },
4128         {
4129           "datasource": {
4130             "type": "prometheus",
4131             "uid": "PBFA97CFB590B2093"
4132           },
4133           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4134           "fieldConfig": {
4135             "defaults": {
4136               "color": {
4137                 "mode": "palette-classic"
4138               },
4139               "custom": {
4140                 "axisLabel": "",
4141                 "axisPlacement": "auto",
4142                 "barAlignment": 0,
4143                 "drawStyle": "line",
4144                 "fillOpacity": 10,
4145                 "gradientMode": "none",
4146                 "hideFrom": {
4147                   "legend": false,
4148                   "tooltip": false,
4149                   "viz": false
4150                 },
4151                 "lineInterpolation": "linear",
4152                 "lineWidth": 1,
4153                 "pointSize": 5,
4154                 "scaleDistribution": {
4155                   "type": "linear"
4156                 },
4157                 "showPoints": "never",
4158                 "spanNulls": true,
4159                 "stacking": {
4160                   "group": "A",
4161                   "mode": "none"
4162                 },
4163                 "thresholdsStyle": {
4164                   "mode": "off"
4165                 }
4166               },
4167               "links": [],
4168               "mappings": [],
4169               "min": 0,
4170               "thresholds": {
4171                 "mode": "absolute",
4172                 "steps": [
4173                   {
4174                     "color": "green"
4175                   },
4176                   {
4177                     "color": "red",
4178                     "value": 80
4179                   }
4180                 ]
4181               },
4182               "unit": "ms"
4183             },
4184             "overrides": []
4185           },
4186           "gridPos": {
4187             "h": 9,
4188             "w": 8,
4189             "x": 8,
4190             "y": 39
4191           },
4192           "id": 72,
4193           "options": {
4194             "legend": {
4195               "calcs": [
4196                 "mean",
4197                 "lastNotNull",
4198                 "max"
4199               ],
4200               "displayMode": "table",
4201               "placement": "bottom",
4202               "showLegend": true
4203             },
4204             "tooltip": {
4205               "mode": "single"
4206             }
4207           },
4208           "pluginVersion": "8.1.3",
4209           "targets": [
4210             {
4211               "datasource": {
4212                 "type": "prometheus",
4213                 "uid": "PBFA97CFB590B2093"
4214               },
4215               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4216               "hide": false,
4217               "legendFormat": "{{instance}} - {{quantile}}",
4218               "refId": "A"
4219             }
4220           ],
4221           "title": "Consumer - ResponseQueueTimeMs",
4222           "type": "timeseries"
4223         },
4224         {
4225           "datasource": {
4226             "type": "prometheus",
4227             "uid": "PBFA97CFB590B2093"
4228           },
4229           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4230           "fieldConfig": {
4231             "defaults": {
4232               "color": {
4233                 "mode": "palette-classic"
4234               },
4235               "custom": {
4236                 "axisLabel": "",
4237                 "axisPlacement": "auto",
4238                 "barAlignment": 0,
4239                 "drawStyle": "line",
4240                 "fillOpacity": 10,
4241                 "gradientMode": "none",
4242                 "hideFrom": {
4243                   "legend": false,
4244                   "tooltip": false,
4245                   "viz": false
4246                 },
4247                 "lineInterpolation": "linear",
4248                 "lineWidth": 1,
4249                 "pointSize": 5,
4250                 "scaleDistribution": {
4251                   "type": "linear"
4252                 },
4253                 "showPoints": "never",
4254                 "spanNulls": true,
4255                 "stacking": {
4256                   "group": "A",
4257                   "mode": "none"
4258                 },
4259                 "thresholdsStyle": {
4260                   "mode": "off"
4261                 }
4262               },
4263               "links": [],
4264               "mappings": [],
4265               "min": 0,
4266               "thresholds": {
4267                 "mode": "absolute",
4268                 "steps": [
4269                   {
4270                     "color": "green"
4271                   },
4272                   {
4273                     "color": "red",
4274                     "value": 80
4275                   }
4276                 ]
4277               },
4278               "unit": "ms"
4279             },
4280             "overrides": []
4281           },
4282           "gridPos": {
4283             "h": 9,
4284             "w": 8,
4285             "x": 16,
4286             "y": 39
4287           },
4288           "id": 73,
4289           "options": {
4290             "legend": {
4291               "calcs": [
4292                 "mean",
4293                 "lastNotNull",
4294                 "max"
4295               ],
4296               "displayMode": "table",
4297               "placement": "bottom",
4298               "showLegend": true
4299             },
4300             "tooltip": {
4301               "mode": "single"
4302             }
4303           },
4304           "pluginVersion": "8.1.3",
4305           "targets": [
4306             {
4307               "datasource": {
4308                 "type": "prometheus",
4309                 "uid": "PBFA97CFB590B2093"
4310               },
4311               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4312               "hide": false,
4313               "legendFormat": "{{instance}} - {{quantile}}",
4314               "refId": "A"
4315             }
4316           ],
4317           "title": "Consumer - ResponseSendTimeMs",
4318           "type": "timeseries"
4319         }
4320       ],
4321       "targets": [
4322         {
4323           "datasource": {
4324             "type": "datasource",
4325             "uid": "grafana"
4326           },
4327           "refId": "A"
4328         }
4329       ],
4330       "title": "Consumer Performance",
4331       "type": "row"
4332     },
4333     {
4334       "collapsed": false,
4335       "datasource": {
4336         "type": "datasource",
4337         "uid": "grafana"
4338       },
4339       "gridPos": {
4340         "h": 1,
4341         "w": 24,
4342         "x": 0,
4343         "y": 98
4344       },
4345       "id": 66,
4346       "panels": [],
4347       "targets": [
4348         {
4349           "datasource": {
4350             "type": "datasource",
4351             "uid": "grafana"
4352           },
4353           "refId": "A"
4354         }
4355       ],
4356       "title": "Fetch Follower Performance",
4357       "type": "row"
4358     },
4359     {
4360       "datasource": {
4361         "type": "prometheus",
4362         "uid": "PBFA97CFB590B2093"
4363       },
4364       "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4365       "fieldConfig": {
4366         "defaults": {
4367           "color": {
4368             "mode": "palette-classic"
4369           },
4370           "custom": {
4371             "axisCenteredZero": false,
4372             "axisColorMode": "text",
4373             "axisLabel": "",
4374             "axisPlacement": "auto",
4375             "barAlignment": 0,
4376             "drawStyle": "line",
4377             "fillOpacity": 10,
4378             "gradientMode": "none",
4379             "hideFrom": {
4380               "legend": false,
4381               "tooltip": false,
4382               "viz": false
4383             },
4384             "lineInterpolation": "linear",
4385             "lineWidth": 1,
4386             "pointSize": 5,
4387             "scaleDistribution": {
4388               "type": "linear"
4389             },
4390             "showPoints": "never",
4391             "spanNulls": true,
4392             "stacking": {
4393               "group": "A",
4394               "mode": "none"
4395             },
4396             "thresholdsStyle": {
4397               "mode": "off"
4398             }
4399           },
4400           "links": [],
4401           "mappings": [],
4402           "min": 0,
4403           "thresholds": {
4404             "mode": "absolute",
4405             "steps": [
4406               {
4407                 "color": "green",
4408                 "value": null
4409               },
4410               {
4411                 "color": "red",
4412                 "value": 80
4413               }
4414             ]
4415           },
4416           "unit": "ms"
4417         },
4418         "overrides": []
4419       },
4420       "gridPos": {
4421         "h": 9,
4422         "w": 12,
4423         "x": 0,
4424         "y": 99
4425       },
4426       "id": 74,
4427       "options": {
4428         "legend": {
4429           "calcs": [
4430             "mean",
4431             "lastNotNull",
4432             "max"
4433           ],
4434           "displayMode": "list",
4435           "placement": "bottom",
4436           "showLegend": true
4437         },
4438         "tooltip": {
4439           "mode": "single",
4440           "sort": "none"
4441         }
4442       },
4443       "pluginVersion": "8.1.3",
4444       "targets": [
4445         {
4446           "datasource": {
4447             "type": "prometheus",
4448             "uid": "PBFA97CFB590B2093"
4449           },
4450           "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4451           "hide": false,
4452           "legendFormat": "{{instance}} - {{quantile}}",
4453           "refId": "A"
4454         }
4455       ],
4456       "title": "FetchFollower - RequestQueueTimeMs",
4457       "type": "timeseries"
4458     },
4459     {
4460       "datasource": {
4461         "type": "prometheus",
4462         "uid": "PBFA97CFB590B2093"
4463       },
4464       "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4465       "fieldConfig": {
4466         "defaults": {
4467           "color": {
4468             "mode": "palette-classic"
4469           },
4470           "custom": {
4471             "axisCenteredZero": false,
4472             "axisColorMode": "text",
4473             "axisLabel": "",
4474             "axisPlacement": "auto",
4475             "barAlignment": 0,
4476             "drawStyle": "line",
4477             "fillOpacity": 10,
4478             "gradientMode": "none",
4479             "hideFrom": {
4480               "legend": false,
4481               "tooltip": false,
4482               "viz": false
4483             },
4484             "lineInterpolation": "linear",
4485             "lineWidth": 1,
4486             "pointSize": 5,
4487             "scaleDistribution": {
4488               "type": "linear"
4489             },
4490             "showPoints": "never",
4491             "spanNulls": true,
4492             "stacking": {
4493               "group": "A",
4494               "mode": "none"
4495             },
4496             "thresholdsStyle": {
4497               "mode": "off"
4498             }
4499           },
4500           "links": [],
4501           "mappings": [],
4502           "min": 0,
4503           "thresholds": {
4504             "mode": "absolute",
4505             "steps": [
4506               {
4507                 "color": "green",
4508                 "value": null
4509               },
4510               {
4511                 "color": "red",
4512                 "value": 80
4513               }
4514             ]
4515           },
4516           "unit": "ms"
4517         },
4518         "overrides": []
4519       },
4520       "gridPos": {
4521         "h": 9,
4522         "w": 12,
4523         "x": 12,
4524         "y": 99
4525       },
4526       "id": 75,
4527       "options": {
4528         "legend": {
4529           "calcs": [
4530             "mean",
4531             "lastNotNull",
4532             "max"
4533           ],
4534           "displayMode": "list",
4535           "placement": "bottom",
4536           "showLegend": true
4537         },
4538         "tooltip": {
4539           "mode": "single",
4540           "sort": "none"
4541         }
4542       },
4543       "pluginVersion": "8.1.3",
4544       "targets": [
4545         {
4546           "datasource": {
4547             "type": "prometheus",
4548             "uid": "PBFA97CFB590B2093"
4549           },
4550           "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4551           "hide": false,
4552           "legendFormat": "{{instance}} - {{quantile}}",
4553           "refId": "A"
4554         }
4555       ],
4556       "title": "FetchFollower - LocalTimeMs",
4557       "type": "timeseries"
4558     },
4559     {
4560       "datasource": {
4561         "type": "prometheus",
4562         "uid": "PBFA97CFB590B2093"
4563       },
4564       "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4565       "fieldConfig": {
4566         "defaults": {
4567           "color": {
4568             "mode": "palette-classic"
4569           },
4570           "custom": {
4571             "axisCenteredZero": false,
4572             "axisColorMode": "text",
4573             "axisLabel": "",
4574             "axisPlacement": "auto",
4575             "barAlignment": 0,
4576             "drawStyle": "line",
4577             "fillOpacity": 10,
4578             "gradientMode": "none",
4579             "hideFrom": {
4580               "legend": false,
4581               "tooltip": false,
4582               "viz": false
4583             },
4584             "lineInterpolation": "linear",
4585             "lineWidth": 1,
4586             "pointSize": 5,
4587             "scaleDistribution": {
4588               "type": "linear"
4589             },
4590             "showPoints": "never",
4591             "spanNulls": true,
4592             "stacking": {
4593               "group": "A",
4594               "mode": "none"
4595             },
4596             "thresholdsStyle": {
4597               "mode": "off"
4598             }
4599           },
4600           "links": [],
4601           "mappings": [],
4602           "thresholds": {
4603             "mode": "absolute",
4604             "steps": [
4605               {
4606                 "color": "green",
4607                 "value": null
4608               },
4609               {
4610                 "color": "red",
4611                 "value": 80
4612               }
4613             ]
4614           },
4615           "unit": "ms"
4616         },
4617         "overrides": []
4618       },
4619       "gridPos": {
4620         "h": 9,
4621         "w": 8,
4622         "x": 0,
4623         "y": 108
4624       },
4625       "id": 76,
4626       "options": {
4627         "legend": {
4628           "calcs": [
4629             "mean",
4630             "lastNotNull",
4631             "max"
4632           ],
4633           "displayMode": "table",
4634           "placement": "bottom",
4635           "showLegend": true
4636         },
4637         "tooltip": {
4638           "mode": "single",
4639           "sort": "none"
4640         }
4641       },
4642       "pluginVersion": "8.1.3",
4643       "targets": [
4644         {
4645           "datasource": {
4646             "type": "prometheus",
4647             "uid": "PBFA97CFB590B2093"
4648           },
4649           "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4650           "hide": false,
4651           "legendFormat": "{{instance}} - {{quantile}}",
4652           "refId": "A"
4653         }
4654       ],
4655       "title": "FetchFollower - RemoteTimeMs",
4656       "type": "timeseries"
4657     },
4658     {
4659       "datasource": {
4660         "type": "prometheus",
4661         "uid": "PBFA97CFB590B2093"
4662       },
4663       "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4664       "fieldConfig": {
4665         "defaults": {
4666           "color": {
4667             "mode": "palette-classic"
4668           },
4669           "custom": {
4670             "axisCenteredZero": false,
4671             "axisColorMode": "text",
4672             "axisLabel": "",
4673             "axisPlacement": "auto",
4674             "barAlignment": 0,
4675             "drawStyle": "line",
4676             "fillOpacity": 10,
4677             "gradientMode": "none",
4678             "hideFrom": {
4679               "legend": false,
4680               "tooltip": false,
4681               "viz": false
4682             },
4683             "lineInterpolation": "linear",
4684             "lineWidth": 1,
4685             "pointSize": 5,
4686             "scaleDistribution": {
4687               "type": "linear"
4688             },
4689             "showPoints": "never",
4690             "spanNulls": true,
4691             "stacking": {
4692               "group": "A",
4693               "mode": "none"
4694             },
4695             "thresholdsStyle": {
4696               "mode": "off"
4697             }
4698           },
4699           "links": [],
4700           "mappings": [],
4701           "thresholds": {
4702             "mode": "absolute",
4703             "steps": [
4704               {
4705                 "color": "green",
4706                 "value": null
4707               },
4708               {
4709                 "color": "red",
4710                 "value": 80
4711               }
4712             ]
4713           },
4714           "unit": "ms"
4715         },
4716         "overrides": []
4717       },
4718       "gridPos": {
4719         "h": 9,
4720         "w": 8,
4721         "x": 8,
4722         "y": 108
4723       },
4724       "id": 77,
4725       "options": {
4726         "legend": {
4727           "calcs": [
4728             "mean",
4729             "lastNotNull",
4730             "max"
4731           ],
4732           "displayMode": "table",
4733           "placement": "bottom",
4734           "showLegend": true
4735         },
4736         "tooltip": {
4737           "mode": "single",
4738           "sort": "none"
4739         }
4740       },
4741       "pluginVersion": "8.1.3",
4742       "targets": [
4743         {
4744           "datasource": {
4745             "type": "prometheus",
4746             "uid": "PBFA97CFB590B2093"
4747           },
4748           "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4749           "hide": false,
4750           "legendFormat": "{{instance}} - {{quantile}}",
4751           "refId": "A"
4752         }
4753       ],
4754       "title": "FetchFollower - ResponseQueueTimeMs",
4755       "type": "timeseries"
4756     },
4757     {
4758       "datasource": {
4759         "type": "prometheus",
4760         "uid": "PBFA97CFB590B2093"
4761       },
4762       "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4763       "fieldConfig": {
4764         "defaults": {
4765           "color": {
4766             "mode": "palette-classic"
4767           },
4768           "custom": {
4769             "axisCenteredZero": false,
4770             "axisColorMode": "text",
4771             "axisLabel": "",
4772             "axisPlacement": "auto",
4773             "barAlignment": 0,
4774             "drawStyle": "line",
4775             "fillOpacity": 10,
4776             "gradientMode": "none",
4777             "hideFrom": {
4778               "legend": false,
4779               "tooltip": false,
4780               "viz": false
4781             },
4782             "lineInterpolation": "linear",
4783             "lineWidth": 1,
4784             "pointSize": 5,
4785             "scaleDistribution": {
4786               "type": "linear"
4787             },
4788             "showPoints": "never",
4789             "spanNulls": true,
4790             "stacking": {
4791               "group": "A",
4792               "mode": "none"
4793             },
4794             "thresholdsStyle": {
4795               "mode": "off"
4796             }
4797           },
4798           "links": [],
4799           "mappings": [],
4800           "min": 0,
4801           "thresholds": {
4802             "mode": "absolute",
4803             "steps": [
4804               {
4805                 "color": "green",
4806                 "value": null
4807               },
4808               {
4809                 "color": "red",
4810                 "value": 80
4811               }
4812             ]
4813           },
4814           "unit": "ms"
4815         },
4816         "overrides": []
4817       },
4818       "gridPos": {
4819         "h": 9,
4820         "w": 8,
4821         "x": 16,
4822         "y": 108
4823       },
4824       "id": 78,
4825       "options": {
4826         "legend": {
4827           "calcs": [
4828             "mean",
4829             "lastNotNull",
4830             "max"
4831           ],
4832           "displayMode": "table",
4833           "placement": "bottom",
4834           "showLegend": true
4835         },
4836         "tooltip": {
4837           "mode": "single",
4838           "sort": "none"
4839         }
4840       },
4841       "pluginVersion": "8.1.3",
4842       "targets": [
4843         {
4844           "datasource": {
4845             "type": "prometheus",
4846             "uid": "PBFA97CFB590B2093"
4847           },
4848           "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4849           "hide": false,
4850           "legendFormat": "{{instance}} - {{quantile}}",
4851           "refId": "A"
4852         }
4853       ],
4854       "title": "FetchFollower - ResponseSendTimeMs",
4855       "type": "timeseries"
4856     },
4857     {
4858       "collapsed": false,
4859       "datasource": {
4860         "type": "datasource",
4861         "uid": "grafana"
4862       },
4863       "gridPos": {
4864         "h": 1,
4865         "w": 24,
4866         "x": 0,
4867         "y": 117
4868       },
4869       "id": 102,
4870       "panels": [],
4871       "targets": [
4872         {
4873           "datasource": {
4874             "type": "datasource",
4875             "uid": "grafana"
4876           },
4877           "refId": "A"
4878         }
4879       ],
4880       "title": "Connections",
4881       "type": "row"
4882     },
4883     {
4884       "datasource": {
4885         "type": "prometheus",
4886         "uid": "PBFA97CFB590B2093"
4887       },
4888       "fieldConfig": {
4889         "defaults": {
4890           "color": {
4891             "mode": "palette-classic"
4892           },
4893           "custom": {
4894             "axisCenteredZero": false,
4895             "axisColorMode": "text",
4896             "axisLabel": "",
4897             "axisPlacement": "auto",
4898             "barAlignment": 0,
4899             "drawStyle": "line",
4900             "fillOpacity": 10,
4901             "gradientMode": "none",
4902             "hideFrom": {
4903               "legend": false,
4904               "tooltip": false,
4905               "viz": false
4906             },
4907             "lineInterpolation": "linear",
4908             "lineWidth": 1,
4909             "pointSize": 5,
4910             "scaleDistribution": {
4911               "type": "linear"
4912             },
4913             "showPoints": "never",
4914             "spanNulls": true,
4915             "stacking": {
4916               "group": "A",
4917               "mode": "none"
4918             },
4919             "thresholdsStyle": {
4920               "mode": "off"
4921             }
4922           },
4923           "links": [],
4924           "mappings": [],
4925           "thresholds": {
4926             "mode": "absolute",
4927             "steps": [
4928               {
4929                 "color": "green",
4930                 "value": null
4931               },
4932               {
4933                 "color": "red",
4934                 "value": 80
4935               }
4936             ]
4937           },
4938           "unit": "short"
4939         },
4940         "overrides": []
4941       },
4942       "gridPos": {
4943         "h": 8,
4944         "w": 12,
4945         "x": 0,
4946         "y": 118
4947       },
4948       "id": 98,
4949       "options": {
4950         "legend": {
4951           "calcs": [
4952             "mean",
4953             "lastNotNull",
4954             "max"
4955           ],
4956           "displayMode": "table",
4957           "placement": "bottom",
4958           "showLegend": true
4959         },
4960         "tooltip": {
4961           "mode": "single",
4962           "sort": "none"
4963         }
4964       },
4965       "pluginVersion": "8.1.3",
4966       "targets": [
4967         {
4968           "datasource": {
4969             "type": "prometheus",
4970             "uid": "PBFA97CFB590B2093"
4971           },
4972           "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
4973           "interval": "",
4974           "legendFormat": "{{listener}}",
4975           "refId": "A"
4976         }
4977       ],
4978       "title": "Connections count per listener",
4979       "type": "timeseries"
4980     },
4981     {
4982       "datasource": {
4983         "type": "prometheus",
4984         "uid": "PBFA97CFB590B2093"
4985       },
4986       "fieldConfig": {
4987         "defaults": {
4988           "color": {
4989             "mode": "palette-classic"
4990           },
4991           "custom": {
4992             "axisCenteredZero": false,
4993             "axisColorMode": "text",
4994             "axisLabel": "",
4995             "axisPlacement": "auto",
4996             "barAlignment": 0,
4997             "drawStyle": "line",
4998             "fillOpacity": 10,
4999             "gradientMode": "none",
5000             "hideFrom": {
5001               "legend": false,
5002               "tooltip": false,
5003               "viz": false
5004             },
5005             "lineInterpolation": "linear",
5006             "lineWidth": 1,
5007             "pointSize": 5,
5008             "scaleDistribution": {
5009               "type": "linear"
5010             },
5011             "showPoints": "never",
5012             "spanNulls": true,
5013             "stacking": {
5014               "group": "A",
5015               "mode": "none"
5016             },
5017             "thresholdsStyle": {
5018               "mode": "off"
5019             }
5020           },
5021           "links": [],
5022           "mappings": [],
5023           "thresholds": {
5024             "mode": "absolute",
5025             "steps": [
5026               {
5027                 "color": "green",
5028                 "value": null
5029               },
5030               {
5031                 "color": "red",
5032                 "value": 80
5033               }
5034             ]
5035           },
5036           "unit": "short"
5037         },
5038         "overrides": []
5039       },
5040       "gridPos": {
5041         "h": 8,
5042         "w": 12,
5043         "x": 12,
5044         "y": 118
5045       },
5046       "id": 100,
5047       "options": {
5048         "legend": {
5049           "calcs": [
5050             "mean",
5051             "lastNotNull",
5052             "max"
5053           ],
5054           "displayMode": "table",
5055           "placement": "bottom",
5056           "showLegend": true
5057         },
5058         "tooltip": {
5059           "mode": "single",
5060           "sort": "none"
5061         }
5062       },
5063       "pluginVersion": "8.1.3",
5064       "targets": [
5065         {
5066           "datasource": {
5067             "type": "prometheus",
5068             "uid": "PBFA97CFB590B2093"
5069           },
5070           "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5071           "interval": "",
5072           "legendFormat": "{{instance}}",
5073           "refId": "A"
5074         }
5075       ],
5076       "title": "Connections count per broker",
5077       "type": "timeseries"
5078     },
5079     {
5080       "datasource": {
5081         "type": "prometheus",
5082         "uid": "PBFA97CFB590B2093"
5083       },
5084       "fieldConfig": {
5085         "defaults": {
5086           "color": {
5087             "mode": "palette-classic"
5088           },
5089           "custom": {
5090             "axisCenteredZero": false,
5091             "axisColorMode": "text",
5092             "axisLabel": "",
5093             "axisPlacement": "auto",
5094             "barAlignment": 0,
5095             "drawStyle": "line",
5096             "fillOpacity": 10,
5097             "gradientMode": "none",
5098             "hideFrom": {
5099               "legend": false,
5100               "tooltip": false,
5101               "viz": false
5102             },
5103             "lineInterpolation": "linear",
5104             "lineWidth": 1,
5105             "pointSize": 5,
5106             "scaleDistribution": {
5107               "type": "linear"
5108             },
5109             "showPoints": "never",
5110             "spanNulls": true,
5111             "stacking": {
5112               "group": "A",
5113               "mode": "none"
5114             },
5115             "thresholdsStyle": {
5116               "mode": "off"
5117             }
5118           },
5119           "links": [],
5120           "mappings": [],
5121           "thresholds": {
5122             "mode": "absolute",
5123             "steps": [
5124               {
5125                 "color": "green",
5126                 "value": null
5127               },
5128               {
5129                 "color": "red",
5130                 "value": 80
5131               }
5132             ]
5133           },
5134           "unit": "short"
5135         },
5136         "overrides": []
5137       },
5138       "gridPos": {
5139         "h": 8,
5140         "w": 12,
5141         "x": 0,
5142         "y": 126
5143       },
5144       "id": 104,
5145       "options": {
5146         "legend": {
5147           "calcs": [
5148             "mean",
5149             "lastNotNull",
5150             "max"
5151           ],
5152           "displayMode": "table",
5153           "placement": "bottom",
5154           "showLegend": true
5155         },
5156         "tooltip": {
5157           "mode": "single",
5158           "sort": "none"
5159         }
5160       },
5161       "pluginVersion": "8.1.3",
5162       "targets": [
5163         {
5164           "datasource": {
5165             "type": "prometheus",
5166             "uid": "PBFA97CFB590B2093"
5167           },
5168           "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5169           "interval": "",
5170           "legendFormat": "{{listener}}",
5171           "refId": "A"
5172         }
5173       ],
5174       "title": "Connections creation rate per listener",
5175       "type": "timeseries"
5176     },
5177     {
5178       "datasource": {
5179         "type": "prometheus",
5180         "uid": "PBFA97CFB590B2093"
5181       },
5182       "fieldConfig": {
5183         "defaults": {
5184           "color": {
5185             "mode": "palette-classic"
5186           },
5187           "custom": {
5188             "axisCenteredZero": false,
5189             "axisColorMode": "text",
5190             "axisLabel": "",
5191             "axisPlacement": "auto",
5192             "barAlignment": 0,
5193             "drawStyle": "line",
5194             "fillOpacity": 10,
5195             "gradientMode": "none",
5196             "hideFrom": {
5197               "legend": false,
5198               "tooltip": false,
5199               "viz": false
5200             },
5201             "lineInterpolation": "linear",
5202             "lineWidth": 1,
5203             "pointSize": 5,
5204             "scaleDistribution": {
5205               "type": "linear"
5206             },
5207             "showPoints": "never",
5208             "spanNulls": true,
5209             "stacking": {
5210               "group": "A",
5211               "mode": "none"
5212             },
5213             "thresholdsStyle": {
5214               "mode": "off"
5215             }
5216           },
5217           "links": [],
5218           "mappings": [],
5219           "thresholds": {
5220             "mode": "absolute",
5221             "steps": [
5222               {
5223                 "color": "green",
5224                 "value": null
5225               },
5226               {
5227                 "color": "red",
5228                 "value": 80
5229               }
5230             ]
5231           },
5232           "unit": "short"
5233         },
5234         "overrides": []
5235       },
5236       "gridPos": {
5237         "h": 8,
5238         "w": 12,
5239         "x": 12,
5240         "y": 126
5241       },
5242       "id": 106,
5243       "options": {
5244         "legend": {
5245           "calcs": [
5246             "mean",
5247             "lastNotNull",
5248             "max"
5249           ],
5250           "displayMode": "table",
5251           "placement": "bottom",
5252           "showLegend": true
5253         },
5254         "tooltip": {
5255           "mode": "single",
5256           "sort": "none"
5257         }
5258       },
5259       "pluginVersion": "8.1.3",
5260       "targets": [
5261         {
5262           "datasource": {
5263             "type": "prometheus",
5264             "uid": "PBFA97CFB590B2093"
5265           },
5266           "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5267           "interval": "",
5268           "legendFormat": "{{instance}}",
5269           "refId": "A"
5270         }
5271       ],
5272       "title": "Connections creation rate per instance",
5273       "type": "timeseries"
5274     },
5275     {
5276       "datasource": {
5277         "type": "prometheus",
5278         "uid": "PBFA97CFB590B2093"
5279       },
5280       "fieldConfig": {
5281         "defaults": {
5282           "color": {
5283             "mode": "palette-classic"
5284           },
5285           "custom": {
5286             "axisCenteredZero": false,
5287             "axisColorMode": "text",
5288             "axisLabel": "",
5289             "axisPlacement": "auto",
5290             "barAlignment": 0,
5291             "drawStyle": "line",
5292             "fillOpacity": 10,
5293             "gradientMode": "none",
5294             "hideFrom": {
5295               "legend": false,
5296               "tooltip": false,
5297               "viz": false
5298             },
5299             "lineInterpolation": "linear",
5300             "lineWidth": 1,
5301             "pointSize": 5,
5302             "scaleDistribution": {
5303               "type": "linear"
5304             },
5305             "showPoints": "never",
5306             "spanNulls": true,
5307             "stacking": {
5308               "group": "A",
5309               "mode": "none"
5310             },
5311             "thresholdsStyle": {
5312               "mode": "off"
5313             }
5314           },
5315           "links": [],
5316           "mappings": [],
5317           "thresholds": {
5318             "mode": "absolute",
5319             "steps": [
5320               {
5321                 "color": "green",
5322                 "value": null
5323               },
5324               {
5325                 "color": "red",
5326                 "value": 80
5327               }
5328             ]
5329           },
5330           "unit": "short"
5331         },
5332         "overrides": []
5333       },
5334       "gridPos": {
5335         "h": 8,
5336         "w": 12,
5337         "x": 0,
5338         "y": 134
5339       },
5340       "id": 108,
5341       "options": {
5342         "legend": {
5343           "calcs": [
5344             "mean",
5345             "lastNotNull",
5346             "max"
5347           ],
5348           "displayMode": "table",
5349           "placement": "bottom",
5350           "showLegend": true
5351         },
5352         "tooltip": {
5353           "mode": "single",
5354           "sort": "none"
5355         }
5356       },
5357       "pluginVersion": "8.1.3",
5358       "targets": [
5359         {
5360           "datasource": {
5361             "type": "prometheus",
5362             "uid": "PBFA97CFB590B2093"
5363           },
5364           "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5365           "interval": "",
5366           "legendFormat": "{{listener}}",
5367           "refId": "A"
5368         }
5369       ],
5370       "title": "Connections close rate per listener",
5371       "type": "timeseries"
5372     },
5373     {
5374       "datasource": {
5375         "type": "prometheus",
5376         "uid": "PBFA97CFB590B2093"
5377       },
5378       "fieldConfig": {
5379         "defaults": {
5380           "color": {
5381             "mode": "palette-classic"
5382           },
5383           "custom": {
5384             "axisCenteredZero": false,
5385             "axisColorMode": "text",
5386             "axisLabel": "",
5387             "axisPlacement": "auto",
5388             "barAlignment": 0,
5389             "drawStyle": "line",
5390             "fillOpacity": 10,
5391             "gradientMode": "none",
5392             "hideFrom": {
5393               "legend": false,
5394               "tooltip": false,
5395               "viz": false
5396             },
5397             "lineInterpolation": "linear",
5398             "lineWidth": 1,
5399             "pointSize": 5,
5400             "scaleDistribution": {
5401               "type": "linear"
5402             },
5403             "showPoints": "never",
5404             "spanNulls": true,
5405             "stacking": {
5406               "group": "A",
5407               "mode": "none"
5408             },
5409             "thresholdsStyle": {
5410               "mode": "off"
5411             }
5412           },
5413           "links": [],
5414           "mappings": [],
5415           "thresholds": {
5416             "mode": "absolute",
5417             "steps": [
5418               {
5419                 "color": "green",
5420                 "value": null
5421               },
5422               {
5423                 "color": "red",
5424                 "value": 80
5425               }
5426             ]
5427           },
5428           "unit": "short"
5429         },
5430         "overrides": []
5431       },
5432       "gridPos": {
5433         "h": 8,
5434         "w": 12,
5435         "x": 12,
5436         "y": 134
5437       },
5438       "id": 110,
5439       "options": {
5440         "legend": {
5441           "calcs": [
5442             "mean",
5443             "lastNotNull",
5444             "max"
5445           ],
5446           "displayMode": "table",
5447           "placement": "bottom",
5448           "showLegend": true
5449         },
5450         "tooltip": {
5451           "mode": "single",
5452           "sort": "none"
5453         }
5454       },
5455       "pluginVersion": "8.1.3",
5456       "targets": [
5457         {
5458           "datasource": {
5459             "type": "prometheus",
5460             "uid": "PBFA97CFB590B2093"
5461           },
5462           "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5463           "interval": "",
5464           "legendFormat": "{{instance}}",
5465           "refId": "A"
5466         }
5467       ],
5468       "title": "Connections close rate per instance",
5469       "type": "timeseries"
5470     },
5471     {
5472       "datasource": {
5473         "type": "prometheus",
5474         "uid": "PBFA97CFB590B2093"
5475       },
5476       "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5477       "fieldConfig": {
5478         "defaults": {
5479           "color": {
5480             "mode": "palette-classic"
5481           },
5482           "custom": {
5483             "axisCenteredZero": false,
5484             "axisColorMode": "text",
5485             "axisLabel": "",
5486             "axisPlacement": "auto",
5487             "barAlignment": 0,
5488             "drawStyle": "line",
5489             "fillOpacity": 10,
5490             "gradientMode": "none",
5491             "hideFrom": {
5492               "legend": false,
5493               "tooltip": false,
5494               "viz": false
5495             },
5496             "lineInterpolation": "linear",
5497             "lineWidth": 1,
5498             "pointSize": 5,
5499             "scaleDistribution": {
5500               "type": "linear"
5501             },
5502             "showPoints": "never",
5503             "spanNulls": true,
5504             "stacking": {
5505               "group": "A",
5506               "mode": "none"
5507             },
5508             "thresholdsStyle": {
5509               "mode": "off"
5510             }
5511           },
5512           "links": [],
5513           "mappings": [],
5514           "min": 0,
5515           "thresholds": {
5516             "mode": "absolute",
5517             "steps": [
5518               {
5519                 "color": "green",
5520                 "value": null
5521               },
5522               {
5523                 "color": "red",
5524                 "value": 80
5525               }
5526             ]
5527           },
5528           "unit": "percent"
5529         },
5530         "overrides": []
5531       },
5532       "gridPos": {
5533         "h": 8,
5534         "w": 12,
5535         "x": 0,
5536         "y": 142
5537       },
5538       "id": 112,
5539       "options": {
5540         "legend": {
5541           "calcs": [
5542             "mean",
5543             "lastNotNull",
5544             "max"
5545           ],
5546           "displayMode": "table",
5547           "placement": "bottom",
5548           "showLegend": true
5549         },
5550         "tooltip": {
5551           "mode": "single",
5552           "sort": "none"
5553         }
5554       },
5555       "pluginVersion": "8.1.3",
5556       "targets": [
5557         {
5558           "datasource": {
5559             "type": "prometheus",
5560             "uid": "PBFA97CFB590B2093"
5561           },
5562           "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5563           "interval": "",
5564           "legendFormat": "{{instance}} - {{listener}}",
5565           "refId": "A"
5566         }
5567       ],
5568       "title": "Acceptor Blocked Percentage",
5569       "type": "timeseries"
5570     },
5571     {
5572       "datasource": {
5573         "type": "prometheus",
5574         "uid": "PBFA97CFB590B2093"
5575       },
5576       "fieldConfig": {
5577         "defaults": {
5578           "color": {
5579             "mode": "palette-classic"
5580           },
5581           "custom": {
5582             "axisCenteredZero": false,
5583             "axisColorMode": "text",
5584             "axisLabel": "",
5585             "axisPlacement": "auto",
5586             "barAlignment": 0,
5587             "drawStyle": "line",
5588             "fillOpacity": 10,
5589             "gradientMode": "none",
5590             "hideFrom": {
5591               "legend": false,
5592               "tooltip": false,
5593               "viz": false
5594             },
5595             "lineInterpolation": "linear",
5596             "lineWidth": 1,
5597             "pointSize": 5,
5598             "scaleDistribution": {
5599               "type": "linear"
5600             },
5601             "showPoints": "never",
5602             "spanNulls": true,
5603             "stacking": {
5604               "group": "A",
5605               "mode": "none"
5606             },
5607             "thresholdsStyle": {
5608               "mode": "off"
5609             }
5610           },
5611           "links": [],
5612           "mappings": [],
5613           "thresholds": {
5614             "mode": "absolute",
5615             "steps": [
5616               {
5617                 "color": "green",
5618                 "value": null
5619               },
5620               {
5621                 "color": "red",
5622                 "value": 80
5623               }
5624             ]
5625           },
5626           "unit": "short"
5627         },
5628         "overrides": []
5629       },
5630       "gridPos": {
5631         "h": 8,
5632         "w": 12,
5633         "x": 12,
5634         "y": 142
5635       },
5636       "id": 114,
5637       "options": {
5638         "legend": {
5639           "calcs": [
5640             "mean",
5641             "lastNotNull",
5642             "max"
5643           ],
5644           "displayMode": "table",
5645           "placement": "bottom",
5646           "showLegend": true
5647         },
5648         "tooltip": {
5649           "mode": "single",
5650           "sort": "none"
5651         }
5652       },
5653       "pluginVersion": "8.1.3",
5654       "targets": [
5655         {
5656           "datasource": {
5657             "type": "prometheus",
5658             "uid": "PBFA97CFB590B2093"
5659           },
5660           "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5661           "interval": "",
5662           "legendFormat": "{{client_software_name}} {{client_software_version}}",
5663           "refId": "A"
5664         }
5665       ],
5666       "title": "Connections per client version",
5667       "type": "timeseries"
5668     },
5669     {
5670       "collapsed": false,
5671       "datasource": {
5672         "type": "datasource",
5673         "uid": "grafana"
5674       },
5675       "gridPos": {
5676         "h": 1,
5677         "w": 24,
5678         "x": 0,
5679         "y": 150
5680       },
5681       "id": 120,
5682       "panels": [],
5683       "targets": [
5684         {
5685           "datasource": {
5686             "type": "datasource",
5687             "uid": "grafana"
5688           },
5689           "refId": "A"
5690         }
5691       ],
5692       "title": "Group Coordinator",
5693       "type": "row"
5694     },
5695     {
5696       "datasource": {
5697         "type": "prometheus",
5698         "uid": "PBFA97CFB590B2093"
5699       },
5700       "description": "Number of consumer groups per group coordinator",
5701       "fieldConfig": {
5702         "defaults": {
5703           "color": {
5704             "mode": "palette-classic"
5705           },
5706           "custom": {
5707             "axisCenteredZero": false,
5708             "axisColorMode": "text",
5709             "axisLabel": "",
5710             "axisPlacement": "auto",
5711             "barAlignment": 0,
5712             "drawStyle": "line",
5713             "fillOpacity": 10,
5714             "gradientMode": "none",
5715             "hideFrom": {
5716               "legend": false,
5717               "tooltip": false,
5718               "viz": false
5719             },
5720             "lineInterpolation": "linear",
5721             "lineWidth": 1,
5722             "pointSize": 5,
5723             "scaleDistribution": {
5724               "type": "linear"
5725             },
5726             "showPoints": "never",
5727             "spanNulls": true,
5728             "stacking": {
5729               "group": "A",
5730               "mode": "none"
5731             },
5732             "thresholdsStyle": {
5733               "mode": "off"
5734             }
5735           },
5736           "links": [],
5737           "mappings": [],
5738           "thresholds": {
5739             "mode": "absolute",
5740             "steps": [
5741               {
5742                 "color": "green",
5743                 "value": null
5744               },
5745               {
5746                 "color": "red",
5747                 "value": 80
5748               }
5749             ]
5750           },
5751           "unit": "short"
5752         },
5753         "overrides": []
5754       },
5755       "gridPos": {
5756         "h": 8,
5757         "w": 12,
5758         "x": 0,
5759         "y": 151
5760       },
5761       "id": 116,
5762       "options": {
5763         "legend": {
5764           "calcs": [
5765             "mean",
5766             "lastNotNull",
5767             "max"
5768           ],
5769           "displayMode": "table",
5770           "placement": "bottom",
5771           "showLegend": true
5772         },
5773         "tooltip": {
5774           "mode": "single",
5775           "sort": "none"
5776         }
5777       },
5778       "pluginVersion": "8.1.3",
5779       "targets": [
5780         {
5781           "datasource": {
5782             "type": "prometheus",
5783             "uid": "PBFA97CFB590B2093"
5784           },
5785           "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5786           "instant": false,
5787           "interval": "",
5788           "legendFormat": "{{instance}}",
5789           "refId": "A"
5790         }
5791       ],
5792       "title": "Consumer groups number per coordinator",
5793       "type": "timeseries"
5794     },
5795     {
5796       "datasource": {
5797         "type": "prometheus",
5798         "uid": "PBFA97CFB590B2093"
5799       },
5800       "description": "Number of consumer group per state",
5801       "fieldConfig": {
5802         "defaults": {
5803           "color": {
5804             "mode": "palette-classic"
5805           },
5806           "custom": {
5807             "axisCenteredZero": false,
5808             "axisColorMode": "text",
5809             "axisLabel": "",
5810             "axisPlacement": "auto",
5811             "barAlignment": 0,
5812             "drawStyle": "line",
5813             "fillOpacity": 10,
5814             "gradientMode": "none",
5815             "hideFrom": {
5816               "legend": false,
5817               "tooltip": false,
5818               "viz": false
5819             },
5820             "lineInterpolation": "linear",
5821             "lineWidth": 1,
5822             "pointSize": 5,
5823             "scaleDistribution": {
5824               "type": "linear"
5825             },
5826             "showPoints": "never",
5827             "spanNulls": true,
5828             "stacking": {
5829               "group": "A",
5830               "mode": "none"
5831             },
5832             "thresholdsStyle": {
5833               "mode": "off"
5834             }
5835           },
5836           "links": [],
5837           "mappings": [],
5838           "thresholds": {
5839             "mode": "absolute",
5840             "steps": [
5841               {
5842                 "color": "green",
5843                 "value": null
5844               },
5845               {
5846                 "color": "red",
5847                 "value": 80
5848               }
5849             ]
5850           },
5851           "unit": "short"
5852         },
5853         "overrides": []
5854       },
5855       "gridPos": {
5856         "h": 8,
5857         "w": 12,
5858         "x": 12,
5859         "y": 151
5860       },
5861       "id": 118,
5862       "options": {
5863         "legend": {
5864           "calcs": [
5865             "mean",
5866             "lastNotNull",
5867             "max"
5868           ],
5869           "displayMode": "table",
5870           "placement": "bottom",
5871           "showLegend": true
5872         },
5873         "tooltip": {
5874           "mode": "single",
5875           "sort": "none"
5876         }
5877       },
5878       "pluginVersion": "8.1.3",
5879       "targets": [
5880         {
5881           "datasource": {
5882             "type": "prometheus",
5883             "uid": "PBFA97CFB590B2093"
5884           },
5885           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5886           "instant": false,
5887           "interval": "",
5888           "legendFormat": "stable",
5889           "refId": "A"
5890         },
5891         {
5892           "datasource": {
5893             "type": "prometheus",
5894             "uid": "PBFA97CFB590B2093"
5895           },
5896           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5897           "interval": "",
5898           "legendFormat": "preparing-rebalance",
5899           "refId": "B"
5900         },
5901         {
5902           "datasource": {
5903             "type": "prometheus",
5904             "uid": "PBFA97CFB590B2093"
5905           },
5906           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5907           "interval": "",
5908           "legendFormat": "dead",
5909           "refId": "C"
5910         },
5911         {
5912           "datasource": {
5913             "type": "prometheus",
5914             "uid": "PBFA97CFB590B2093"
5915           },
5916           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5917           "interval": "",
5918           "legendFormat": "completing-rebalance",
5919           "refId": "D"
5920         },
5921         {
5922           "datasource": {
5923             "type": "prometheus",
5924             "uid": "PBFA97CFB590B2093"
5925           },
5926           "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
5927           "interval": "",
5928           "legendFormat": "empty",
5929           "refId": "E"
5930         }
5931       ],
5932       "title": "Nb consumer groups per state",
5933       "type": "timeseries"
5934     },
5935     {
5936       "collapsed": false,
5937       "datasource": {
5938         "type": "datasource",
5939         "uid": "grafana"
5940       },
5941       "gridPos": {
5942         "h": 1,
5943         "w": 24,
5944         "x": 0,
5945         "y": 159
5946       },
5947       "id": 46,
5948       "panels": [],
5949       "targets": [
5950         {
5951           "datasource": {
5952             "type": "datasource",
5953             "uid": "grafana"
5954           },
5955           "refId": "A"
5956         }
5957       ],
5958       "title": "Message Conversion",
5959       "type": "row"
5960     },
5961     {
5962       "datasource": {
5963         "type": "prometheus",
5964         "uid": "PBFA97CFB590B2093"
5965       },
5966       "description": "The number of messages produced converted to match the log.message.format.version.",
5967       "fieldConfig": {
5968         "defaults": {
5969           "color": {
5970             "mode": "palette-classic"
5971           },
5972           "custom": {
5973             "axisCenteredZero": false,
5974             "axisColorMode": "text",
5975             "axisLabel": "",
5976             "axisPlacement": "auto",
5977             "barAlignment": 0,
5978             "drawStyle": "line",
5979             "fillOpacity": 10,
5980             "gradientMode": "none",
5981             "hideFrom": {
5982               "legend": false,
5983               "tooltip": false,
5984               "viz": false
5985             },
5986             "lineInterpolation": "linear",
5987             "lineWidth": 1,
5988             "pointSize": 5,
5989             "scaleDistribution": {
5990               "type": "linear"
5991             },
5992             "showPoints": "never",
5993             "spanNulls": true,
5994             "stacking": {
5995               "group": "A",
5996               "mode": "none"
5997             },
5998             "thresholdsStyle": {
5999               "mode": "off"
6000             }
6001           },
6002           "links": [],
6003           "mappings": [],
6004           "min": 0,
6005           "thresholds": {
6006             "mode": "absolute",
6007             "steps": [
6008               {
6009                 "color": "green",
6010                 "value": null
6011               },
6012               {
6013                 "color": "red",
6014                 "value": 80
6015               }
6016             ]
6017           },
6018           "unit": "ops"
6019         },
6020         "overrides": []
6021       },
6022       "gridPos": {
6023         "h": 8,
6024         "w": 8,
6025         "x": 0,
6026         "y": 160
6027       },
6028       "id": 48,
6029       "options": {
6030         "legend": {
6031           "calcs": [
6032             "mean",
6033             "lastNotNull",
6034             "max"
6035           ],
6036           "displayMode": "list",
6037           "placement": "bottom",
6038           "showLegend": true
6039         },
6040         "tooltip": {
6041           "mode": "single",
6042           "sort": "none"
6043         }
6044       },
6045       "pluginVersion": "8.1.3",
6046       "targets": [
6047         {
6048           "datasource": {
6049             "type": "prometheus",
6050             "uid": "PBFA97CFB590B2093"
6051           },
6052           "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6053           "hide": false,
6054           "interval": "",
6055           "legendFormat": "{{topic}}",
6056           "refId": "A"
6057         }
6058       ],
6059       "title": "Number of produced message conversion",
6060       "type": "timeseries"
6061     },
6062     {
6063       "datasource": {
6064         "type": "prometheus",
6065         "uid": "PBFA97CFB590B2093"
6066       },
6067       "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
6068       "fieldConfig": {
6069         "defaults": {
6070           "color": {
6071             "mode": "palette-classic"
6072           },
6073           "custom": {
6074             "axisCenteredZero": false,
6075             "axisColorMode": "text",
6076             "axisLabel": "",
6077             "axisPlacement": "auto",
6078             "barAlignment": 0,
6079             "drawStyle": "line",
6080             "fillOpacity": 10,
6081             "gradientMode": "none",
6082             "hideFrom": {
6083               "legend": false,
6084               "tooltip": false,
6085               "viz": false
6086             },
6087             "lineInterpolation": "linear",
6088             "lineWidth": 1,
6089             "pointSize": 5,
6090             "scaleDistribution": {
6091               "type": "linear"
6092             },
6093             "showPoints": "never",
6094             "spanNulls": true,
6095             "stacking": {
6096               "group": "A",
6097               "mode": "none"
6098             },
6099             "thresholdsStyle": {
6100               "mode": "off"
6101             }
6102           },
6103           "links": [],
6104           "mappings": [],
6105           "min": 0,
6106           "thresholds": {
6107             "mode": "absolute",
6108             "steps": [
6109               {
6110                 "color": "green",
6111                 "value": null
6112               },
6113               {
6114                 "color": "red",
6115                 "value": 80
6116               }
6117             ]
6118           },
6119           "unit": "ops"
6120         },
6121         "overrides": []
6122       },
6123       "gridPos": {
6124         "h": 8,
6125         "w": 8,
6126         "x": 8,
6127         "y": 160
6128       },
6129       "id": 51,
6130       "options": {
6131         "legend": {
6132           "calcs": [
6133             "mean",
6134             "lastNotNull",
6135             "max"
6136           ],
6137           "displayMode": "list",
6138           "placement": "bottom",
6139           "showLegend": true
6140         },
6141         "tooltip": {
6142           "mode": "single",
6143           "sort": "none"
6144         }
6145       },
6146       "pluginVersion": "8.1.3",
6147       "targets": [
6148         {
6149           "datasource": {
6150             "type": "prometheus",
6151             "uid": "PBFA97CFB590B2093"
6152           },
6153           "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6154           "hide": false,
6155           "interval": "",
6156           "legendFormat": "{{topic}}",
6157           "refId": "A"
6158         }
6159       ],
6160       "title": "Number of consumed message conversion",
6161       "type": "timeseries"
6162     },
6163     {
6164       "datasource": {
6165         "type": "datasource",
6166         "uid": "grafana"
6167       },
6168       "description": "Number of connection per client version",
6169       "fieldConfig": {
6170         "defaults": {
6171           "color": {
6172             "mode": "palette-classic"
6173           },
6174           "custom": {
6175             "hideFrom": {
6176               "legend": false,
6177               "tooltip": false,
6178               "viz": false
6179             }
6180           },
6181           "decimals": 0,
6182           "mappings": [],
6183           "unit": "short"
6184         },
6185         "overrides": []
6186       },
6187       "gridPos": {
6188         "h": 8,
6189         "w": 8,
6190         "x": 16,
6191         "y": 160
6192       },
6193       "id": 96,
6194       "links": [],
6195       "options": {
6196         "legend": {
6197           "calcs": [],
6198           "displayMode": "table",
6199           "placement": "bottom",
6200           "showLegend": true,
6201           "values": [
6202             "value"
6203           ]
6204         },
6205         "pieType": "pie",
6206         "reduceOptions": {
6207           "calcs": [
6208             "lastNotNull"
6209           ],
6210           "fields": "",
6211           "values": false
6212         },
6213         "tooltip": {
6214           "mode": "single",
6215           "sort": "none"
6216         }
6217       },
6218       "targets": [
6219         {
6220           "datasource": {
6221             "type": "datasource",
6222             "uid": "grafana"
6223           },
6224           "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6225           "interval": "",
6226           "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6227           "refId": "A"
6228         }
6229       ],
6230       "title": "Client version repartition",
6231       "type": "piechart"
6232     }
6233   ],
6234   "refresh": "5s",
6235   "schemaVersion": 38,
6236   "style": "dark",
6237   "tags": [],
6238   "templating": {
6239     "list": [
6240       {
6241         "current": {
6242           "selected": false,
6243           "text": "dev",
6244           "value": "dev"
6245         },
6246         "datasource": {
6247           "type": "prometheus",
6248           "uid": "PBFA97CFB590B2093"
6249         },
6250         "definition": "label_values(env)",
6251         "hide": 0,
6252         "includeAll": false,
6253         "label": "Environment",
6254         "multi": false,
6255         "name": "env",
6256         "options": [],
6257         "query": {
6258           "query": "label_values(env)",
6259           "refId": "Prometheus-env-Variable-Query"
6260         },
6261         "refresh": 1,
6262         "regex": "",
6263         "skipUrlSync": false,
6264         "sort": 0,
6265         "tagValuesQuery": "",
6266         "tagsQuery": "",
6267         "type": "query",
6268         "useTags": false
6269       },
6270       {
6271         "current": {
6272           "selected": true,
6273           "text": [
6274             "All"
6275           ],
6276           "value": [
6277             "$__all"
6278           ]
6279         },
6280         "datasource": {
6281           "type": "prometheus",
6282           "uid": "PBFA97CFB590B2093"
6283         },
6284         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6285         "hide": 0,
6286         "includeAll": true,
6287         "label": "Instance",
6288         "multi": true,
6289         "name": "instance",
6290         "options": [],
6291         "query": {
6292           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6293           "refId": "Prometheus-instance-Variable-Query"
6294         },
6295         "refresh": 1,
6296         "regex": "",
6297         "skipUrlSync": false,
6298         "sort": 0,
6299         "tagValuesQuery": "",
6300         "tagsQuery": "",
6301         "type": "query",
6302         "useTags": false
6303       },
6304       {
6305         "allValue": "",
6306         "current": {
6307           "selected": true,
6308           "text": [
6309             "0.95"
6310           ],
6311           "value": [
6312             "0.95"
6313           ]
6314         },
6315         "datasource": {
6316           "type": "prometheus",
6317           "uid": "PBFA97CFB590B2093"
6318         },
6319         "definition": "label_values(quantile)",
6320         "hide": 0,
6321         "includeAll": true,
6322         "label": "Percentile",
6323         "multi": true,
6324         "name": "percentile",
6325         "options": [],
6326         "query": {
6327           "query": "label_values(quantile)",
6328           "refId": "Prometheus-percentile-Variable-Query"
6329         },
6330         "refresh": 1,
6331         "regex": "",
6332         "skipUrlSync": false,
6333         "sort": 0,
6334         "tagValuesQuery": "",
6335         "tagsQuery": "",
6336         "type": "query",
6337         "useTags": false
6338       }
6339     ]
6340   },
6341   "time": {
6342     "from": "now-5m",
6343     "to": "now"
6344   },
6345   "timepicker": {
6346     "refresh_intervals": [
6347       "5s",
6348       "10s",
6349       "30s",
6350       "1m",
6351       "5m",
6352       "15m",
6353       "30m",
6354       "1h",
6355       "2h",
6356       "1d"
6357     ],
6358     "time_options": [
6359       "5m",
6360       "15m",
6361       "1h",
6362       "6h",
6363       "12h",
6364       "24h",
6365       "2d",
6366       "7d",
6367       "30d"
6368     ]
6369   },
6370   "timezone": "browser",
6371   "title": "Kafka cluster",
6372   "uid": "qu-QZdfZz",
6373   "version": 1,
6374   "weekStart": ""
6375 }