juplo.de Git - demos/kafka/monitoring/blob - kafka-brokers.json

   1 {
   2   "annotations": {
   3     "list": [
   4       {
   5         "builtIn": 1,
   6         "datasource": {
   7           "type": "datasource",
   8           "uid": "grafana"
   9         },
  10         "enable": true,
  11         "hide": true,
  12         "iconColor": "rgba(0, 211, 255, 1)",
  13         "name": "Annotations & Alerts",
  14         "target": {
  15           "limit": 100,
  16           "matchAny": false,
  17           "tags": [],
  18           "type": "dashboard"
  19         },
  20         "type": "dashboard"
  21       }
  22     ]
  23   },
  24   "description": "Kafka resource usage and throughput",
  25   "editable": true,
  26   "fiscalYearStartMonth": 0,
  27   "gnetId": 721,
  28   "graphTooltip": 0,
  29   "links": [],
  30   "liveNow": false,
  31   "panels": [
  32     {
  33       "collapsed": false,
  34       "datasource": {
  35         "type": "datasource",
  36         "uid": "grafana"
  37       },
  38       "gridPos": {
  39         "h": 1,
  40         "w": 24,
  41         "x": 0,
  42         "y": 0
  43       },
  44       "id": 42,
  45       "panels": [],
  46       "targets": [
  47         {
  48           "datasource": {
  49             "type": "datasource",
  50             "uid": "grafana"
  51           },
  52           "refId": "A"
  53         }
  54       ],
  55       "title": "Healthcheck",
  56       "type": "row"
  57     },
  58     {
  59       "datasource": {
  60         "type": "prometheus",
  61         "uid": "PBFA97CFB590B2093"
  62       },
  63       "description": "Number of active controllers in the cluster.",
  64       "fieldConfig": {
  65         "defaults": {
  66           "mappings": [
  67             {
  68               "options": {
  69                 "match": "null",
  70                 "result": {
  71                   "text": "N/A"
  72                 }
  73               },
  74               "type": "special"
  75             }
  76           ],
  77           "thresholds": {
  78             "mode": "absolute",
  79             "steps": [
  80               {
  81                 "color": "#299c46",
  82                 "value": null
  83               },
  84               {
  85                 "color": "#e5ac0e",
  86                 "value": 2
  87               },
  88               {
  89                 "color": "#bf1b00"
  90               }
  91             ]
  92           },
  93           "unit": "none"
  94         },
  95         "overrides": []
  96       },
  97       "gridPos": {
  98         "h": 4,
  99         "w": 4,
 100         "x": 0,
 101         "y": 1
 102       },
 103       "id": 12,
 104       "links": [],
 105       "maxDataPoints": 100,
 106       "options": {
 107         "colorMode": "value",
 108         "graphMode": "none",
 109         "justifyMode": "auto",
 110         "orientation": "vertical",
 111         "reduceOptions": {
 112           "calcs": [
 113             "lastNotNull"
 114           ],
 115           "fields": "",
 116           "values": false
 117         },
 118         "text": {},
 119         "textMode": "value_and_name"
 120       },
 121       "pluginVersion": "10.0.3",
 122       "targets": [
 123         {
 124           "datasource": {
 125             "type": "prometheus",
 126             "uid": "PBFA97CFB590B2093"
 127           },
 128           "expr": "kafka_controller_kafkacontroller_activecontrollercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"} > 0",
 129           "format": "time_series",
 130           "instant": true,
 131           "interval": "",
 132           "intervalFactor": 1,
 133           "legendFormat": "{{instance}}",
 134           "refId": "A"
 135         }
 136       ],
 137       "title": "Active Controllers",
 138       "type": "stat"
 139     },
 140     {
 141       "datasource": {
 142         "type": "prometheus",
 143         "uid": "PBFA97CFB590B2093"
 144       },
 145       "description": "Number of partitions under min insync replicas.",
 146       "fieldConfig": {
 147         "defaults": {
 148           "color": {
 149             "mode": "thresholds"
 150           },
 151           "mappings": [
 152             {
 153               "options": {
 154                 "match": "null",
 155                 "result": {
 156                   "text": "N/A"
 157                 }
 158               },
 159               "type": "special"
 160             }
 161           ],
 162           "thresholds": {
 163             "mode": "absolute",
 164             "steps": [
 165               {
 166                 "color": "#508642",
 167                 "value": null
 168               },
 169               {
 170                 "color": "rgba(237, 129, 40, 0.89)",
 171                 "value": 1
 172               },
 173               {
 174                 "color": "#bf1b00",
 175                 "value": 5
 176               }
 177             ]
 178           },
 179           "unit": "none"
 180         },
 181         "overrides": []
 182       },
 183       "gridPos": {
 184         "h": 4,
 185         "w": 4,
 186         "x": 4,
 187         "y": 1
 188       },
 189       "id": 32,
 190       "links": [],
 191       "maxDataPoints": 100,
 192       "options": {
 193         "colorMode": "value",
 194         "graphMode": "none",
 195         "justifyMode": "auto",
 196         "orientation": "horizontal",
 197         "reduceOptions": {
 198           "calcs": [
 199             "lastNotNull"
 200           ],
 201           "fields": "",
 202           "values": false
 203         },
 204         "text": {},
 205         "textMode": "auto"
 206       },
 207       "pluginVersion": "10.0.3",
 208       "targets": [
 209         {
 210           "datasource": {
 211             "type": "prometheus",
 212             "uid": "PBFA97CFB590B2093"
 213           },
 214           "expr": "sum(kafka_cluster_partition_underminisr{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 215           "format": "time_series",
 216           "hide": false,
 217           "instant": true,
 218           "interval": "",
 219           "intervalFactor": 2,
 220           "legendFormat": "",
 221           "refId": "A"
 222         }
 223       ],
 224       "title": "Under Min ISR Partitions",
 225       "type": "stat"
 226     },
 227     {
 228       "datasource": {
 229         "type": "prometheus",
 230         "uid": "PBFA97CFB590B2093"
 231       },
 232       "description": "Number of partitions that dont have an active leader and are hence not writable or readable.",
 233       "fieldConfig": {
 234         "defaults": {
 235           "color": {
 236             "mode": "thresholds"
 237           },
 238           "mappings": [
 239             {
 240               "options": {
 241                 "match": "null",
 242                 "result": {
 243                   "text": "N/A"
 244                 }
 245               },
 246               "type": "special"
 247             }
 248           ],
 249           "thresholds": {
 250             "mode": "absolute",
 251             "steps": [
 252               {
 253                 "color": "#508642",
 254                 "value": null
 255               },
 256               {
 257                 "color": "#ef843c",
 258                 "value": 1
 259               },
 260               {
 261                 "color": "#bf1b00",
 262                 "value": 1
 263               }
 264             ]
 265           },
 266           "unit": "none"
 267         },
 268         "overrides": []
 269       },
 270       "gridPos": {
 271         "h": 4,
 272         "w": 4,
 273         "x": 8,
 274         "y": 1
 275       },
 276       "id": 22,
 277       "links": [],
 278       "maxDataPoints": 100,
 279       "options": {
 280         "colorMode": "value",
 281         "graphMode": "none",
 282         "justifyMode": "auto",
 283         "orientation": "horizontal",
 284         "reduceOptions": {
 285           "calcs": [
 286             "lastNotNull"
 287           ],
 288           "fields": "",
 289           "values": false
 290         },
 291         "text": {},
 292         "textMode": "auto"
 293       },
 294       "pluginVersion": "10.0.3",
 295       "targets": [
 296         {
 297           "datasource": {
 298             "type": "prometheus",
 299             "uid": "PBFA97CFB590B2093"
 300           },
 301           "expr": "sum(kafka_controller_kafkacontroller_offlinepartitionscount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 302           "format": "time_series",
 303           "instant": true,
 304           "interval": "",
 305           "intervalFactor": 1,
 306           "legendFormat": "",
 307           "refId": "A"
 308         }
 309       ],
 310       "title": "Offline Partitions Count",
 311       "type": "stat"
 312     },
 313     {
 314       "datasource": {
 315         "type": "prometheus",
 316         "uid": "PBFA97CFB590B2093"
 317       },
 318       "description": "",
 319       "fieldConfig": {
 320         "defaults": {
 321           "color": {
 322             "mode": "thresholds"
 323           },
 324           "mappings": [
 325             {
 326               "options": {
 327                 "match": "null",
 328                 "result": {
 329                   "text": "N/A"
 330                 }
 331               },
 332               "type": "special"
 333             }
 334           ],
 335           "thresholds": {
 336             "mode": "absolute",
 337             "steps": [
 338               {
 339                 "color": "#299c46",
 340                 "value": null
 341               },
 342               {
 343                 "color": "rgba(237, 129, 40, 0.89)",
 344                 "value": 2
 345               },
 346               {
 347                 "color": "#d44a3a"
 348               }
 349             ]
 350           },
 351           "unit": "none"
 352         },
 353         "overrides": []
 354       },
 355       "gridPos": {
 356         "h": 4,
 357         "w": 4,
 358         "x": 12,
 359         "y": 1
 360       },
 361       "id": 33,
 362       "links": [],
 363       "maxDataPoints": 100,
 364       "options": {
 365         "colorMode": "value",
 366         "graphMode": "none",
 367         "justifyMode": "auto",
 368         "orientation": "horizontal",
 369         "reduceOptions": {
 370           "calcs": [
 371             "lastNotNull"
 372           ],
 373           "fields": "",
 374           "values": false
 375         },
 376         "text": {},
 377         "textMode": "auto"
 378       },
 379       "pluginVersion": "10.0.3",
 380       "targets": [
 381         {
 382           "datasource": {
 383             "type": "prometheus",
 384             "uid": "PBFA97CFB590B2093"
 385           },
 386           "expr": "sum(kafka_controller_kafkacontroller_preferredreplicaimbalancecount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 387           "format": "time_series",
 388           "instant": true,
 389           "interval": "",
 390           "intervalFactor": 1,
 391           "legendFormat": "",
 392           "refId": "A"
 393         }
 394       ],
 395       "title": "Preferred Replica Imbalance",
 396       "type": "stat"
 397     },
 398     {
 399       "datasource": {
 400         "type": "prometheus",
 401         "uid": "PBFA97CFB590B2093"
 402       },
 403       "fieldConfig": {
 404         "defaults": {
 405           "color": {
 406             "mode": "palette-classic"
 407           },
 408           "custom": {
 409             "axisCenteredZero": false,
 410             "axisColorMode": "text",
 411             "axisLabel": "Bytes/s",
 412             "axisPlacement": "auto",
 413             "barAlignment": 0,
 414             "drawStyle": "line",
 415             "fillOpacity": 10,
 416             "gradientMode": "none",
 417             "hideFrom": {
 418               "legend": false,
 419               "tooltip": false,
 420               "viz": false
 421             },
 422             "lineInterpolation": "linear",
 423             "lineWidth": 2,
 424             "pointSize": 5,
 425             "scaleDistribution": {
 426               "type": "linear"
 427             },
 428             "showPoints": "never",
 429             "spanNulls": false,
 430             "stacking": {
 431               "group": "A",
 432               "mode": "none"
 433             },
 434             "thresholdsStyle": {
 435               "mode": "off"
 436             }
 437           },
 438           "links": [],
 439           "mappings": [],
 440           "thresholds": {
 441             "mode": "absolute",
 442             "steps": [
 443               {
 444                 "color": "green",
 445                 "value": null
 446               },
 447               {
 448                 "color": "red",
 449                 "value": 80
 450               }
 451             ]
 452           },
 453           "unit": "Bps"
 454         },
 455         "overrides": []
 456       },
 457       "gridPos": {
 458         "h": 8,
 459         "w": 8,
 460         "x": 16,
 461         "y": 1
 462       },
 463       "id": 84,
 464       "links": [],
 465       "options": {
 466         "legend": {
 467           "calcs": [
 468             "mean",
 469             "lastNotNull",
 470             "max"
 471           ],
 472           "displayMode": "list",
 473           "placement": "bottom",
 474           "showLegend": true
 475         },
 476         "tooltip": {
 477           "mode": "single",
 478           "sort": "none"
 479         }
 480       },
 481       "pluginVersion": "8.1.3",
 482       "targets": [
 483         {
 484           "datasource": {
 485             "type": "prometheus",
 486             "uid": "PBFA97CFB590B2093"
 487           },
 488           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 489           "format": "time_series",
 490           "interval": "",
 491           "intervalFactor": 2,
 492           "legendFormat": "Bytes in",
 493           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 494           "refId": "A",
 495           "step": 4
 496         },
 497         {
 498           "datasource": {
 499             "type": "prometheus",
 500             "uid": "PBFA97CFB590B2093"
 501           },
 502           "expr": "sum(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
 503           "format": "time_series",
 504           "hide": false,
 505           "interval": "",
 506           "intervalFactor": 2,
 507           "legendFormat": "Bytes out",
 508           "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
 509           "refId": "B",
 510           "step": 4
 511         }
 512       ],
 513       "title": "Broker network throughput",
 514       "type": "timeseries"
 515     },
 516     {
 517       "datasource": {
 518         "type": "prometheus",
 519         "uid": "PBFA97CFB590B2093"
 520       },
 521       "description": "Number of under-replicated partitions (| ISR | < | all replicas |).",
 522       "fieldConfig": {
 523         "defaults": {
 524           "color": {
 525             "mode": "thresholds"
 526           },
 527           "mappings": [
 528             {
 529               "options": {
 530                 "match": "null",
 531                 "result": {
 532                   "text": "N/A"
 533                 }
 534               },
 535               "type": "special"
 536             }
 537           ],
 538           "thresholds": {
 539             "mode": "absolute",
 540             "steps": [
 541               {
 542                 "color": "#508642",
 543                 "value": null
 544               },
 545               {
 546                 "color": "rgba(237, 129, 40, 0.89)",
 547                 "value": 1
 548               },
 549               {
 550                 "color": "#bf1b00",
 551                 "value": 5
 552               }
 553             ]
 554           },
 555           "unit": "none"
 556         },
 557         "overrides": []
 558       },
 559       "gridPos": {
 560         "h": 4,
 561         "w": 4,
 562         "x": 0,
 563         "y": 5
 564       },
 565       "id": 20,
 566       "links": [],
 567       "maxDataPoints": 100,
 568       "options": {
 569         "colorMode": "value",
 570         "graphMode": "none",
 571         "justifyMode": "auto",
 572         "orientation": "horizontal",
 573         "reduceOptions": {
 574           "calcs": [
 575             "lastNotNull"
 576           ],
 577           "fields": "",
 578           "values": false
 579         },
 580         "text": {},
 581         "textMode": "auto"
 582       },
 583       "pluginVersion": "10.0.3",
 584       "targets": [
 585         {
 586           "datasource": {
 587             "type": "prometheus",
 588             "uid": "PBFA97CFB590B2093"
 589           },
 590           "expr": "sum(kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 591           "format": "time_series",
 592           "hide": false,
 593           "instant": true,
 594           "interval": "",
 595           "intervalFactor": 2,
 596           "legendFormat": "",
 597           "refId": "A"
 598         }
 599       ],
 600       "title": "Under Replicated Partitions",
 601       "type": "stat"
 602     },
 603     {
 604       "datasource": {
 605         "type": "prometheus",
 606         "uid": "PBFA97CFB590B2093"
 607       },
 608       "description": "Number of Brokers Online",
 609       "fieldConfig": {
 610         "defaults": {
 611           "color": {
 612             "mode": "thresholds"
 613           },
 614           "mappings": [
 615             {
 616               "options": {
 617                 "match": "null",
 618                 "result": {
 619                   "text": "N/A"
 620                 }
 621               },
 622               "type": "special"
 623             }
 624           ],
 625           "thresholds": {
 626             "mode": "absolute",
 627             "steps": [
 628               {
 629                 "color": "#d44a3a",
 630                 "value": null
 631               },
 632               {
 633                 "color": "rgba(237, 129, 40, 0.89)",
 634                 "value": 0
 635               },
 636               {
 637                 "color": "#299c46",
 638                 "value": 2
 639               }
 640             ]
 641           },
 642           "unit": "none"
 643         },
 644         "overrides": []
 645       },
 646       "gridPos": {
 647         "h": 4,
 648         "w": 4,
 649         "x": 4,
 650         "y": 5
 651       },
 652       "id": 14,
 653       "links": [],
 654       "maxDataPoints": 100,
 655       "options": {
 656         "colorMode": "value",
 657         "graphMode": "none",
 658         "justifyMode": "auto",
 659         "orientation": "horizontal",
 660         "reduceOptions": {
 661           "calcs": [
 662             "lastNotNull"
 663           ],
 664           "fields": "",
 665           "values": false
 666         },
 667         "text": {},
 668         "textMode": "auto"
 669       },
 670       "pluginVersion": "10.0.3",
 671       "repeatDirection": "h",
 672       "targets": [
 673         {
 674           "datasource": {
 675             "type": "prometheus",
 676             "uid": "PBFA97CFB590B2093"
 677           },
 678           "expr": "count(kafka_server_replicamanager_leadercount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 679           "format": "time_series",
 680           "instant": true,
 681           "interval": "",
 682           "intervalFactor": 1,
 683           "legendFormat": "",
 684           "refId": "A"
 685         }
 686       ],
 687       "title": "Brokers Online",
 688       "type": "stat"
 689     },
 690     {
 691       "datasource": {
 692         "type": "prometheus",
 693         "uid": "PBFA97CFB590B2093"
 694       },
 695       "description": "Partitions that are online",
 696       "fieldConfig": {
 697         "defaults": {
 698           "color": {
 699             "mode": "thresholds"
 700           },
 701           "mappings": [
 702             {
 703               "options": {
 704                 "match": "null",
 705                 "result": {
 706                   "text": "N/A"
 707                 }
 708               },
 709               "type": "special"
 710             }
 711           ],
 712           "thresholds": {
 713             "mode": "absolute",
 714             "steps": [
 715               {
 716                 "color": "#d44a3a",
 717                 "value": null
 718               },
 719               {
 720                 "color": "rgba(237, 129, 40, 0.89)",
 721                 "value": 0
 722               },
 723               {
 724                 "color": "#299c46",
 725                 "value": 0
 726               }
 727             ]
 728           },
 729           "unit": "none"
 730         },
 731         "overrides": []
 732       },
 733       "gridPos": {
 734         "h": 4,
 735         "w": 4,
 736         "x": 8,
 737         "y": 5
 738       },
 739       "id": 18,
 740       "links": [],
 741       "maxDataPoints": 100,
 742       "options": {
 743         "colorMode": "value",
 744         "graphMode": "none",
 745         "justifyMode": "auto",
 746         "orientation": "horizontal",
 747         "reduceOptions": {
 748           "calcs": [
 749             "lastNotNull"
 750           ],
 751           "fields": "",
 752           "values": false
 753         },
 754         "text": {},
 755         "textMode": "auto"
 756       },
 757       "pluginVersion": "10.0.3",
 758       "targets": [
 759         {
 760           "datasource": {
 761             "type": "prometheus",
 762             "uid": "PBFA97CFB590B2093"
 763           },
 764           "expr": "sum(kafka_server_replicamanager_partitioncount{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 765           "format": "time_series",
 766           "instant": true,
 767           "interval": "",
 768           "intervalFactor": 1,
 769           "legendFormat": "",
 770           "refId": "A"
 771         }
 772       ],
 773       "title": "Online Partitions",
 774       "type": "stat"
 775     },
 776     {
 777       "datasource": {
 778         "type": "prometheus",
 779         "uid": "PBFA97CFB590B2093"
 780       },
 781       "description": "Unclean leader election rate",
 782       "fieldConfig": {
 783         "defaults": {
 784           "color": {
 785             "mode": "thresholds"
 786           },
 787           "mappings": [
 788             {
 789               "options": {
 790                 "match": "null",
 791                 "result": {
 792                   "text": "N/A"
 793                 }
 794               },
 795               "type": "special"
 796             }
 797           ],
 798           "thresholds": {
 799             "mode": "absolute",
 800             "steps": [
 801               {
 802                 "color": "#299c46",
 803                 "value": null
 804               },
 805               {
 806                 "color": "rgba(237, 129, 40, 0.89)",
 807                 "value": 2
 808               },
 809               {
 810                 "color": "#d44a3a"
 811               }
 812             ]
 813           },
 814           "unit": "none"
 815         },
 816         "overrides": []
 817       },
 818       "gridPos": {
 819         "h": 4,
 820         "w": 4,
 821         "x": 12,
 822         "y": 5
 823       },
 824       "id": 16,
 825       "links": [],
 826       "maxDataPoints": 100,
 827       "options": {
 828         "colorMode": "value",
 829         "graphMode": "none",
 830         "justifyMode": "auto",
 831         "orientation": "horizontal",
 832         "reduceOptions": {
 833           "calcs": [
 834             "lastNotNull"
 835           ],
 836           "fields": "",
 837           "values": false
 838         },
 839         "text": {},
 840         "textMode": "auto"
 841       },
 842       "pluginVersion": "10.0.3",
 843       "targets": [
 844         {
 845           "datasource": {
 846             "type": "prometheus",
 847             "uid": "PBFA97CFB590B2093"
 848           },
 849           "expr": "sum(kafka_controller_controllerstats_uncleanleaderelectionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
 850           "format": "time_series",
 851           "instant": true,
 852           "interval": "",
 853           "intervalFactor": 1,
 854           "legendFormat": "",
 855           "refId": "A"
 856         }
 857       ],
 858       "title": "Unclean Leader Election Rate",
 859       "type": "stat"
 860     },
 861     {
 862       "datasource": {
 863         "type": "prometheus",
 864         "uid": "PBFA97CFB590B2093"
 865       },
 866       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
 867       "fieldConfig": {
 868         "defaults": {
 869           "color": {
 870             "mode": "palette-classic"
 871           },
 872           "custom": {
 873             "axisCenteredZero": false,
 874             "axisColorMode": "text",
 875             "axisLabel": "",
 876             "axisPlacement": "auto",
 877             "barAlignment": 0,
 878             "drawStyle": "line",
 879             "fillOpacity": 10,
 880             "gradientMode": "none",
 881             "hideFrom": {
 882               "legend": false,
 883               "tooltip": false,
 884               "viz": false
 885             },
 886             "lineInterpolation": "linear",
 887             "lineWidth": 1,
 888             "pointSize": 5,
 889             "scaleDistribution": {
 890               "type": "linear"
 891             },
 892             "showPoints": "never",
 893             "spanNulls": true,
 894             "stacking": {
 895               "group": "A",
 896               "mode": "none"
 897             },
 898             "thresholdsStyle": {
 899               "mode": "off"
 900             }
 901           },
 902           "links": [],
 903           "mappings": [],
 904           "min": 0,
 905           "thresholds": {
 906             "mode": "absolute",
 907             "steps": [
 908               {
 909                 "color": "green",
 910                 "value": null
 911               },
 912               {
 913                 "color": "red",
 914                 "value": 80
 915               }
 916             ]
 917           },
 918           "unit": "ops"
 919         },
 920         "overrides": [
 921           {
 922             "__systemRef": "hideSeriesFrom",
 923             "matcher": {
 924               "id": "byNames",
 925               "options": {
 926                 "mode": "exclude",
 927                 "names": [
 928                   "kafka-3:80"
 929                 ],
 930                 "prefix": "All except:",
 931                 "readOnly": true
 932               }
 933             },
 934             "properties": [
 935               {
 936                 "id": "custom.hideFrom",
 937                 "value": {
 938                   "legend": false,
 939                   "tooltip": false,
 940                   "viz": true
 941                 }
 942               }
 943             ]
 944           }
 945         ]
 946       },
 947       "gridPos": {
 948         "h": 9,
 949         "w": 12,
 950         "x": 12,
 951         "y": 9
 952       },
 953       "id": 123,
 954       "options": {
 955         "legend": {
 956           "calcs": [
 957             "mean",
 958             "lastNotNull",
 959             "max"
 960           ],
 961           "displayMode": "table",
 962           "placement": "bottom",
 963           "showLegend": true
 964         },
 965         "tooltip": {
 966           "mode": "single",
 967           "sort": "none"
 968         }
 969       },
 970       "pluginVersion": "8.1.3",
 971       "targets": [
 972         {
 973           "datasource": {
 974             "type": "prometheus",
 975             "uid": "PBFA97CFB590B2093"
 976           },
 977           "editorMode": "code",
 978           "exemplar": false,
 979           "expr": "kafka_server_replicamanager_underreplicatedpartitions{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
 980           "hide": false,
 981           "interval": "",
 982           "legendFormat": "{{instance}}",
 983           "range": true,
 984           "refId": "A"
 985         }
 986       ],
 987       "title": "IsrExpands per Sec",
 988       "type": "timeseries"
 989     },
 990     {
 991       "collapsed": false,
 992       "datasource": {
 993         "type": "datasource",
 994         "uid": "grafana"
 995       },
 996       "gridPos": {
 997         "h": 1,
 998         "w": 24,
 999         "x": 0,
1000         "y": 18
1001       },
1002       "id": 31,
1003       "panels": [],
1004       "targets": [
1005         {
1006           "datasource": {
1007             "type": "datasource",
1008             "uid": "grafana"
1009           },
1010           "refId": "A"
1011         }
1012       ],
1013       "title": "Request rate",
1014       "type": "row"
1015     },
1016     {
1017       "datasource": {
1018         "type": "prometheus",
1019         "uid": "PBFA97CFB590B2093"
1020       },
1021       "description": "Produce request rate.",
1022       "fieldConfig": {
1023         "defaults": {
1024           "mappings": [
1025             {
1026               "options": {
1027                 "match": "null",
1028                 "result": {
1029                   "text": "N/A"
1030                 }
1031               },
1032               "type": "special"
1033             }
1034           ],
1035           "thresholds": {
1036             "mode": "absolute",
1037             "steps": [
1038               {
1039                 "color": "green",
1040                 "value": null
1041               }
1042             ]
1043           },
1044           "unit": "none"
1045         },
1046         "overrides": []
1047       },
1048       "gridPos": {
1049         "h": 4,
1050         "w": 4,
1051         "x": 0,
1052         "y": 19
1053       },
1054       "id": 93,
1055       "links": [],
1056       "maxDataPoints": 100,
1057       "options": {
1058         "colorMode": "value",
1059         "graphMode": "area",
1060         "justifyMode": "auto",
1061         "orientation": "horizontal",
1062         "reduceOptions": {
1063           "calcs": [
1064             "lastNotNull"
1065           ],
1066           "fields": "",
1067           "values": false
1068         },
1069         "text": {},
1070         "textMode": "auto"
1071       },
1072       "pluginVersion": "10.0.3",
1073       "targets": [
1074         {
1075           "datasource": {
1076             "type": "prometheus",
1077             "uid": "PBFA97CFB590B2093"
1078           },
1079           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1080           "interval": "",
1081           "legendFormat": "",
1082           "refId": "A"
1083         }
1084       ],
1085       "title": "All Request Per Sec",
1086       "type": "stat"
1087     },
1088     {
1089       "datasource": {
1090         "type": "prometheus",
1091         "uid": "PBFA97CFB590B2093"
1092       },
1093       "description": "Produce request rate.",
1094       "fieldConfig": {
1095         "defaults": {
1096           "mappings": [
1097             {
1098               "options": {
1099                 "match": "null",
1100                 "result": {
1101                   "text": "N/A"
1102                 }
1103               },
1104               "type": "special"
1105             }
1106           ],
1107           "thresholds": {
1108             "mode": "absolute",
1109             "steps": [
1110               {
1111                 "color": "green",
1112                 "value": null
1113               }
1114             ]
1115           },
1116           "unit": "none"
1117         },
1118         "overrides": []
1119       },
1120       "gridPos": {
1121         "h": 4,
1122         "w": 4,
1123         "x": 4,
1124         "y": 19
1125       },
1126       "id": 35,
1127       "links": [],
1128       "maxDataPoints": 100,
1129       "options": {
1130         "colorMode": "value",
1131         "graphMode": "area",
1132         "justifyMode": "auto",
1133         "orientation": "horizontal",
1134         "reduceOptions": {
1135           "calcs": [
1136             "lastNotNull"
1137           ],
1138           "fields": "",
1139           "values": false
1140         },
1141         "text": {},
1142         "textMode": "auto"
1143       },
1144       "pluginVersion": "10.0.3",
1145       "targets": [
1146         {
1147           "datasource": {
1148             "type": "prometheus",
1149             "uid": "PBFA97CFB590B2093"
1150           },
1151           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Produce\"}[5m]))",
1152           "interval": "",
1153           "legendFormat": "",
1154           "refId": "A"
1155         }
1156       ],
1157       "title": "Produce Request Per Sec",
1158       "type": "stat"
1159     },
1160     {
1161       "datasource": {
1162         "type": "prometheus",
1163         "uid": "PBFA97CFB590B2093"
1164       },
1165       "description": "Fetch request rate.",
1166       "fieldConfig": {
1167         "defaults": {
1168           "mappings": [
1169             {
1170               "options": {
1171                 "match": "null",
1172                 "result": {
1173                   "text": "N/A"
1174                 }
1175               },
1176               "type": "special"
1177             }
1178           ],
1179           "thresholds": {
1180             "mode": "absolute",
1181             "steps": [
1182               {
1183                 "color": "green",
1184                 "value": null
1185               }
1186             ]
1187           },
1188           "unit": "none"
1189         },
1190         "overrides": []
1191       },
1192       "gridPos": {
1193         "h": 4,
1194         "w": 4,
1195         "x": 8,
1196         "y": 19
1197       },
1198       "id": 37,
1199       "links": [],
1200       "maxDataPoints": 100,
1201       "options": {
1202         "colorMode": "value",
1203         "graphMode": "area",
1204         "justifyMode": "auto",
1205         "orientation": "horizontal",
1206         "reduceOptions": {
1207           "calcs": [
1208             "lastNotNull"
1209           ],
1210           "fields": "",
1211           "values": false
1212         },
1213         "text": {},
1214         "textMode": "auto"
1215       },
1216       "pluginVersion": "10.0.3",
1217       "targets": [
1218         {
1219           "datasource": {
1220             "type": "prometheus",
1221             "uid": "PBFA97CFB590B2093"
1222           },
1223           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"FetchConsumer\"}[5m]))",
1224           "interval": "",
1225           "legendFormat": "",
1226           "refId": "A"
1227         }
1228       ],
1229       "title": "Consumer Fetch Request Per Sec",
1230       "type": "stat"
1231     },
1232     {
1233       "datasource": {
1234         "type": "prometheus",
1235         "uid": "PBFA97CFB590B2093"
1236       },
1237       "description": "Fetch request rate.",
1238       "fieldConfig": {
1239         "defaults": {
1240           "mappings": [
1241             {
1242               "options": {
1243                 "match": "null",
1244                 "result": {
1245                   "text": "N/A"
1246                 }
1247               },
1248               "type": "special"
1249             }
1250           ],
1251           "thresholds": {
1252             "mode": "absolute",
1253             "steps": [
1254               {
1255                 "color": "green",
1256                 "value": null
1257               }
1258             ]
1259           },
1260           "unit": "none"
1261         },
1262         "overrides": []
1263       },
1264       "gridPos": {
1265         "h": 4,
1266         "w": 4,
1267         "x": 0,
1268         "y": 23
1269       },
1270       "id": 94,
1271       "links": [],
1272       "maxDataPoints": 100,
1273       "options": {
1274         "colorMode": "value",
1275         "graphMode": "area",
1276         "justifyMode": "auto",
1277         "orientation": "horizontal",
1278         "reduceOptions": {
1279           "calcs": [
1280             "lastNotNull"
1281           ],
1282           "fields": "",
1283           "values": false
1284         },
1285         "text": {},
1286         "textMode": "auto"
1287       },
1288       "pluginVersion": "10.0.3",
1289       "targets": [
1290         {
1291           "datasource": {
1292             "type": "prometheus",
1293             "uid": "PBFA97CFB590B2093"
1294           },
1295           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Fetch\"}[5m]))",
1296           "interval": "",
1297           "legendFormat": "",
1298           "refId": "A"
1299         }
1300       ],
1301       "title": "Broker Fetch Request Per Sec",
1302       "type": "stat"
1303     },
1304     {
1305       "datasource": {
1306         "type": "prometheus",
1307         "uid": "PBFA97CFB590B2093"
1308       },
1309       "description": "Offset Commit request rate.",
1310       "fieldConfig": {
1311         "defaults": {
1312           "mappings": [
1313             {
1314               "options": {
1315                 "match": "null",
1316                 "result": {
1317                   "text": "N/A"
1318                 }
1319               },
1320               "type": "special"
1321             }
1322           ],
1323           "thresholds": {
1324             "mode": "absolute",
1325             "steps": [
1326               {
1327                 "color": "green",
1328                 "value": null
1329               }
1330             ]
1331           },
1332           "unit": "none"
1333         },
1334         "overrides": []
1335       },
1336       "gridPos": {
1337         "h": 4,
1338         "w": 4,
1339         "x": 4,
1340         "y": 23
1341       },
1342       "id": 38,
1343       "links": [],
1344       "maxDataPoints": 100,
1345       "options": {
1346         "colorMode": "value",
1347         "graphMode": "area",
1348         "justifyMode": "auto",
1349         "orientation": "horizontal",
1350         "reduceOptions": {
1351           "calcs": [
1352             "lastNotNull"
1353           ],
1354           "fields": "",
1355           "values": false
1356         },
1357         "text": {},
1358         "textMode": "auto"
1359       },
1360       "pluginVersion": "10.0.3",
1361       "targets": [
1362         {
1363           "datasource": {
1364             "type": "prometheus",
1365             "uid": "PBFA97CFB590B2093"
1366           },
1367           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"OffsetCommit\"}[5m]))",
1368           "interval": "",
1369           "legendFormat": "",
1370           "refId": "A"
1371         }
1372       ],
1373       "title": "Offset Commit Request Per Sec",
1374       "type": "stat"
1375     },
1376     {
1377       "datasource": {
1378         "type": "prometheus",
1379         "uid": "PBFA97CFB590B2093"
1380       },
1381       "description": "Metadata request rate.",
1382       "fieldConfig": {
1383         "defaults": {
1384           "mappings": [
1385             {
1386               "options": {
1387                 "match": "null",
1388                 "result": {
1389                   "text": "N/A"
1390                 }
1391               },
1392               "type": "special"
1393             }
1394           ],
1395           "thresholds": {
1396             "mode": "absolute",
1397             "steps": [
1398               {
1399                 "color": "green",
1400                 "value": null
1401               }
1402             ]
1403           },
1404           "unit": "none"
1405         },
1406         "overrides": []
1407       },
1408       "gridPos": {
1409         "h": 4,
1410         "w": 4,
1411         "x": 8,
1412         "y": 23
1413       },
1414       "id": 36,
1415       "links": [],
1416       "maxDataPoints": 100,
1417       "options": {
1418         "colorMode": "value",
1419         "graphMode": "area",
1420         "justifyMode": "auto",
1421         "orientation": "horizontal",
1422         "reduceOptions": {
1423           "calcs": [
1424             "lastNotNull"
1425           ],
1426           "fields": "",
1427           "values": false
1428         },
1429         "text": {},
1430         "textMode": "auto"
1431       },
1432       "pluginVersion": "10.0.3",
1433       "targets": [
1434         {
1435           "datasource": {
1436             "type": "prometheus",
1437             "uid": "PBFA97CFB590B2093"
1438           },
1439           "expr": "sum(rate(kafka_network_requestmetrics_requestspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",request=\"Metadata\"}[5m]))",
1440           "interval": "",
1441           "legendFormat": "",
1442           "refId": "A"
1443         }
1444       ],
1445       "title": "Metadata Request Per Sec",
1446       "type": "stat"
1447     },
1448     {
1449       "collapsed": false,
1450       "datasource": {
1451         "type": "datasource",
1452         "uid": "grafana"
1453       },
1454       "gridPos": {
1455         "h": 1,
1456         "w": 24,
1457         "x": 0,
1458         "y": 27
1459       },
1460       "id": 40,
1461       "panels": [],
1462       "targets": [
1463         {
1464           "datasource": {
1465             "type": "datasource",
1466             "uid": "grafana"
1467           },
1468           "refId": "A"
1469         }
1470       ],
1471       "title": "System",
1472       "type": "row"
1473     },
1474     {
1475       "datasource": {
1476         "type": "prometheus",
1477         "uid": "PBFA97CFB590B2093"
1478       },
1479       "fieldConfig": {
1480         "defaults": {
1481           "color": {
1482             "mode": "palette-classic"
1483           },
1484           "custom": {
1485             "axisCenteredZero": false,
1486             "axisColorMode": "text",
1487             "axisLabel": "Cores",
1488             "axisPlacement": "auto",
1489             "barAlignment": 0,
1490             "drawStyle": "line",
1491             "fillOpacity": 10,
1492             "gradientMode": "none",
1493             "hideFrom": {
1494               "legend": false,
1495               "tooltip": false,
1496               "viz": false
1497             },
1498             "lineInterpolation": "linear",
1499             "lineWidth": 2,
1500             "pointSize": 5,
1501             "scaleDistribution": {
1502               "type": "linear"
1503             },
1504             "showPoints": "never",
1505             "spanNulls": false,
1506             "stacking": {
1507               "group": "A",
1508               "mode": "none"
1509             },
1510             "thresholdsStyle": {
1511               "mode": "off"
1512             }
1513           },
1514           "links": [],
1515           "mappings": [],
1516           "thresholds": {
1517             "mode": "absolute",
1518             "steps": [
1519               {
1520                 "color": "green",
1521                 "value": null
1522               },
1523               {
1524                 "color": "red",
1525                 "value": 80
1526               }
1527             ]
1528           },
1529           "unit": "percent"
1530         },
1531         "overrides": [
1532           {
1533             "matcher": {
1534               "id": "byName",
1535               "options": "localhost:7071"
1536             },
1537             "properties": [
1538               {
1539                 "id": "color",
1540                 "value": {
1541                   "fixedColor": "#629E51",
1542                   "mode": "fixed"
1543                 }
1544               }
1545             ]
1546           }
1547         ]
1548       },
1549       "gridPos": {
1550         "h": 8,
1551         "w": 8,
1552         "x": 0,
1553         "y": 28
1554       },
1555       "id": 27,
1556       "links": [],
1557       "options": {
1558         "legend": {
1559           "calcs": [
1560             "mean",
1561             "lastNotNull",
1562             "max"
1563           ],
1564           "displayMode": "table",
1565           "placement": "bottom",
1566           "showLegend": true
1567         },
1568         "tooltip": {
1569           "mode": "single",
1570           "sort": "none"
1571         }
1572       },
1573       "pluginVersion": "8.1.3",
1574       "targets": [
1575         {
1576           "datasource": {
1577             "type": "prometheus",
1578             "uid": "PBFA97CFB590B2093"
1579           },
1580           "expr": "irate(process_cpu_seconds_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])*100",
1581           "format": "time_series",
1582           "interval": "",
1583           "intervalFactor": 2,
1584           "legendFormat": "{{instance}}",
1585           "metric": "process_cpu_secondspersec",
1586           "refId": "A",
1587           "step": 4
1588         }
1589       ],
1590       "title": "CPU Usage",
1591       "type": "timeseries"
1592     },
1593     {
1594       "datasource": {
1595         "type": "prometheus",
1596         "uid": "PBFA97CFB590B2093"
1597       },
1598       "fieldConfig": {
1599         "defaults": {
1600           "color": {
1601             "mode": "palette-classic"
1602           },
1603           "custom": {
1604             "axisCenteredZero": false,
1605             "axisColorMode": "text",
1606             "axisLabel": "Memory",
1607             "axisPlacement": "auto",
1608             "barAlignment": 0,
1609             "drawStyle": "line",
1610             "fillOpacity": 10,
1611             "gradientMode": "none",
1612             "hideFrom": {
1613               "legend": false,
1614               "tooltip": false,
1615               "viz": false
1616             },
1617             "lineInterpolation": "linear",
1618             "lineWidth": 2,
1619             "pointSize": 5,
1620             "scaleDistribution": {
1621               "type": "linear"
1622             },
1623             "showPoints": "never",
1624             "spanNulls": false,
1625             "stacking": {
1626               "group": "A",
1627               "mode": "none"
1628             },
1629             "thresholdsStyle": {
1630               "mode": "off"
1631             }
1632           },
1633           "links": [],
1634           "mappings": [],
1635           "thresholds": {
1636             "mode": "absolute",
1637             "steps": [
1638               {
1639                 "color": "green",
1640                 "value": null
1641               },
1642               {
1643                 "color": "red",
1644                 "value": 80
1645               }
1646             ]
1647           },
1648           "unit": "bytes"
1649         },
1650         "overrides": [
1651           {
1652             "matcher": {
1653               "id": "byName",
1654               "options": "localhost:7071"
1655             },
1656             "properties": [
1657               {
1658                 "id": "color",
1659                 "value": {
1660                   "fixedColor": "#BA43A9",
1661                   "mode": "fixed"
1662                 }
1663               }
1664             ]
1665           }
1666         ]
1667       },
1668       "gridPos": {
1669         "h": 8,
1670         "w": 8,
1671         "x": 8,
1672         "y": 28
1673       },
1674       "id": 2,
1675       "links": [],
1676       "options": {
1677         "legend": {
1678           "calcs": [
1679             "mean",
1680             "lastNotNull",
1681             "max"
1682           ],
1683           "displayMode": "table",
1684           "placement": "bottom",
1685           "showLegend": true
1686         },
1687         "tooltip": {
1688           "mode": "single",
1689           "sort": "none"
1690         }
1691       },
1692       "pluginVersion": "8.1.3",
1693       "targets": [
1694         {
1695           "datasource": {
1696             "type": "prometheus",
1697             "uid": "PBFA97CFB590B2093"
1698           },
1699           "expr": "sum without(area)(jvm_memory_bytes_used{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
1700           "interval": "",
1701           "intervalFactor": 2,
1702           "legendFormat": "{{instance}}",
1703           "metric": "jvm_memory_bytes_used",
1704           "refId": "A",
1705           "step": 4
1706         },
1707         {
1708           "datasource": {
1709             "type": "prometheus",
1710             "uid": "PBFA97CFB590B2093"
1711           },
1712           "expr": "jvm_memory_bytes_max{job=\"kafka-broker\",area=\"heap\",env=\"$env\",instance=~\"$instance\"}",
1713           "interval": "",
1714           "legendFormat": "{{instance}}",
1715           "refId": "B"
1716         }
1717       ],
1718       "title": "JVM Memory Used",
1719       "type": "timeseries"
1720     },
1721     {
1722       "datasource": {
1723         "type": "prometheus",
1724         "uid": "PBFA97CFB590B2093"
1725       },
1726       "fieldConfig": {
1727         "defaults": {
1728           "color": {
1729             "mode": "palette-classic"
1730           },
1731           "custom": {
1732             "axisCenteredZero": false,
1733             "axisColorMode": "text",
1734             "axisLabel": "% time in GC",
1735             "axisPlacement": "auto",
1736             "barAlignment": 0,
1737             "drawStyle": "line",
1738             "fillOpacity": 10,
1739             "gradientMode": "none",
1740             "hideFrom": {
1741               "legend": false,
1742               "tooltip": false,
1743               "viz": false
1744             },
1745             "lineInterpolation": "linear",
1746             "lineWidth": 2,
1747             "pointSize": 5,
1748             "scaleDistribution": {
1749               "type": "linear"
1750             },
1751             "showPoints": "never",
1752             "spanNulls": false,
1753             "stacking": {
1754               "group": "A",
1755               "mode": "none"
1756             },
1757             "thresholdsStyle": {
1758               "mode": "off"
1759             }
1760           },
1761           "links": [],
1762           "mappings": [],
1763           "thresholds": {
1764             "mode": "absolute",
1765             "steps": [
1766               {
1767                 "color": "green",
1768                 "value": null
1769               },
1770               {
1771                 "color": "red",
1772                 "value": 80
1773               }
1774             ]
1775           },
1776           "unit": "percentunit"
1777         },
1778         "overrides": [
1779           {
1780             "matcher": {
1781               "id": "byName",
1782               "options": "localhost:7071"
1783             },
1784             "properties": [
1785               {
1786                 "id": "color",
1787                 "value": {
1788                   "fixedColor": "#890F02",
1789                   "mode": "fixed"
1790                 }
1791               }
1792             ]
1793           }
1794         ]
1795       },
1796       "gridPos": {
1797         "h": 8,
1798         "w": 8,
1799         "x": 16,
1800         "y": 28
1801       },
1802       "id": 3,
1803       "links": [],
1804       "options": {
1805         "legend": {
1806           "calcs": [
1807             "mean",
1808             "lastNotNull",
1809             "max"
1810           ],
1811           "displayMode": "table",
1812           "placement": "bottom",
1813           "showLegend": true
1814         },
1815         "tooltip": {
1816           "mode": "single",
1817           "sort": "none"
1818         }
1819       },
1820       "pluginVersion": "8.1.3",
1821       "targets": [
1822         {
1823           "datasource": {
1824             "type": "prometheus",
1825             "uid": "PBFA97CFB590B2093"
1826           },
1827           "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m]))",
1828           "interval": "",
1829           "intervalFactor": 2,
1830           "legendFormat": "{{instance}}",
1831           "metric": "jvm_gc_collection_seconds_sum",
1832           "refId": "A",
1833           "step": 4
1834         }
1835       ],
1836       "title": "Time spent in GC",
1837       "type": "timeseries"
1838     },
1839     {
1840       "collapsed": true,
1841       "datasource": {
1842         "type": "datasource",
1843         "uid": "grafana"
1844       },
1845       "gridPos": {
1846         "h": 1,
1847         "w": 24,
1848         "x": 0,
1849         "y": 36
1850       },
1851       "id": 29,
1852       "panels": [
1853         {
1854           "datasource": {
1855             "type": "prometheus",
1856             "uid": "PBFA97CFB590B2093"
1857           },
1858           "fieldConfig": {
1859             "defaults": {
1860               "color": {
1861                 "mode": "palette-classic"
1862               },
1863               "custom": {
1864                 "axisLabel": "Messages/s",
1865                 "axisPlacement": "auto",
1866                 "barAlignment": 0,
1867                 "drawStyle": "line",
1868                 "fillOpacity": 10,
1869                 "gradientMode": "none",
1870                 "hideFrom": {
1871                   "legend": false,
1872                   "tooltip": false,
1873                   "viz": false
1874                 },
1875                 "lineInterpolation": "linear",
1876                 "lineWidth": 2,
1877                 "pointSize": 5,
1878                 "scaleDistribution": {
1879                   "type": "linear"
1880                 },
1881                 "showPoints": "never",
1882                 "spanNulls": false,
1883                 "stacking": {
1884                   "group": "A",
1885                   "mode": "none"
1886                 },
1887                 "thresholdsStyle": {
1888                   "mode": "off"
1889                 }
1890               },
1891               "links": [],
1892               "mappings": [],
1893               "min": 0,
1894               "thresholds": {
1895                 "mode": "absolute",
1896                 "steps": [
1897                   {
1898                     "color": "green"
1899                   }
1900                 ]
1901               },
1902               "unit": "cps"
1903             },
1904             "overrides": []
1905           },
1906           "gridPos": {
1907             "h": 8,
1908             "w": 8,
1909             "x": 0,
1910             "y": 24
1911           },
1912           "id": 4,
1913           "links": [],
1914           "options": {
1915             "legend": {
1916               "calcs": [
1917                 "mean",
1918                 "lastNotNull",
1919                 "max"
1920               ],
1921               "displayMode": "table",
1922               "placement": "bottom",
1923               "showLegend": true
1924             },
1925             "tooltip": {
1926               "mode": "single"
1927             }
1928           },
1929           "pluginVersion": "8.1.3",
1930           "targets": [
1931             {
1932               "datasource": {
1933                 "type": "prometheus",
1934                 "uid": "PBFA97CFB590B2093"
1935               },
1936               "exemplar": true,
1937               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
1938               "interval": "",
1939               "intervalFactor": 2,
1940               "legendFormat": "bytes/sec",
1941               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
1942               "refId": "A",
1943               "step": 4
1944             }
1945           ],
1946           "title": "Messages In",
1947           "type": "timeseries"
1948         },
1949         {
1950           "datasource": {
1951             "type": "prometheus",
1952             "uid": "PBFA97CFB590B2093"
1953           },
1954           "fieldConfig": {
1955             "defaults": {
1956               "color": {
1957                 "mode": "palette-classic"
1958               },
1959               "custom": {
1960                 "axisLabel": "Bytes/s",
1961                 "axisPlacement": "auto",
1962                 "barAlignment": 0,
1963                 "drawStyle": "line",
1964                 "fillOpacity": 10,
1965                 "gradientMode": "none",
1966                 "hideFrom": {
1967                   "legend": false,
1968                   "tooltip": false,
1969                   "viz": false
1970                 },
1971                 "lineInterpolation": "linear",
1972                 "lineWidth": 2,
1973                 "pointSize": 5,
1974                 "scaleDistribution": {
1975                   "type": "linear"
1976                 },
1977                 "showPoints": "never",
1978                 "spanNulls": false,
1979                 "stacking": {
1980                   "group": "A",
1981                   "mode": "none"
1982                 },
1983                 "thresholdsStyle": {
1984                   "mode": "off"
1985                 }
1986               },
1987               "links": [],
1988               "mappings": [],
1989               "min": 0,
1990               "thresholds": {
1991                 "mode": "absolute",
1992                 "steps": [
1993                   {
1994                     "color": "green"
1995                   },
1996                   {
1997                     "color": "red",
1998                     "value": 80
1999                   }
2000                 ]
2001               },
2002               "unit": "Bps"
2003             },
2004             "overrides": []
2005           },
2006           "gridPos": {
2007             "h": 8,
2008             "w": 8,
2009             "x": 8,
2010             "y": 24
2011           },
2012           "id": 5,
2013           "links": [],
2014           "options": {
2015             "legend": {
2016               "calcs": [
2017                 "mean",
2018                 "lastNotNull",
2019                 "max"
2020               ],
2021               "displayMode": "table",
2022               "placement": "bottom",
2023               "showLegend": true
2024             },
2025             "tooltip": {
2026               "mode": "single"
2027             }
2028           },
2029           "pluginVersion": "8.1.3",
2030           "targets": [
2031             {
2032               "datasource": {
2033                 "type": "prometheus",
2034                 "uid": "PBFA97CFB590B2093"
2035               },
2036               "exemplar": true,
2037               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2038               "format": "time_series",
2039               "interval": "",
2040               "intervalFactor": 2,
2041               "legendFormat": "bytes/sec",
2042               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2043               "refId": "A",
2044               "step": 4
2045             }
2046           ],
2047           "title": "Bytes In",
2048           "type": "timeseries"
2049         },
2050         {
2051           "datasource": {
2052             "type": "prometheus",
2053             "uid": "PBFA97CFB590B2093"
2054           },
2055           "fieldConfig": {
2056             "defaults": {
2057               "color": {
2058                 "mode": "palette-classic"
2059               },
2060               "custom": {
2061                 "axisLabel": "Bytes/s",
2062                 "axisPlacement": "auto",
2063                 "barAlignment": 0,
2064                 "drawStyle": "line",
2065                 "fillOpacity": 10,
2066                 "gradientMode": "none",
2067                 "hideFrom": {
2068                   "legend": false,
2069                   "tooltip": false,
2070                   "viz": false
2071                 },
2072                 "lineInterpolation": "linear",
2073                 "lineWidth": 2,
2074                 "pointSize": 5,
2075                 "scaleDistribution": {
2076                   "type": "linear"
2077                 },
2078                 "showPoints": "never",
2079                 "spanNulls": false,
2080                 "stacking": {
2081                   "group": "A",
2082                   "mode": "none"
2083                 },
2084                 "thresholdsStyle": {
2085                   "mode": "off"
2086                 }
2087               },
2088               "links": [],
2089               "mappings": [],
2090               "min": 0,
2091               "thresholds": {
2092                 "mode": "absolute",
2093                 "steps": [
2094                   {
2095                     "color": "green"
2096                   },
2097                   {
2098                     "color": "red",
2099                     "value": 80
2100                   }
2101                 ]
2102               },
2103               "unit": "Bps"
2104             },
2105             "overrides": []
2106           },
2107           "gridPos": {
2108             "h": 8,
2109             "w": 8,
2110             "x": 16,
2111             "y": 24
2112           },
2113           "id": 6,
2114           "links": [],
2115           "options": {
2116             "legend": {
2117               "calcs": [
2118                 "mean",
2119                 "lastNotNull",
2120                 "max"
2121               ],
2122               "displayMode": "table",
2123               "placement": "bottom",
2124               "showLegend": true
2125             },
2126             "tooltip": {
2127               "mode": "single"
2128             }
2129           },
2130           "pluginVersion": "8.1.3",
2131           "targets": [
2132             {
2133               "datasource": {
2134                 "type": "prometheus",
2135                 "uid": "PBFA97CFB590B2093"
2136               },
2137               "exemplar": true,
2138               "expr": "sum without(instance,topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",topic!=\"\"}[5m]))",
2139               "interval": "",
2140               "intervalFactor": 2,
2141               "legendFormat": "bytes/sec",
2142               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2143               "refId": "A",
2144               "step": 4
2145             }
2146           ],
2147           "title": "Bytes Out",
2148           "type": "timeseries"
2149         },
2150         {
2151           "datasource": {
2152             "type": "prometheus",
2153             "uid": "PBFA97CFB590B2093"
2154           },
2155           "fieldConfig": {
2156             "defaults": {
2157               "color": {
2158                 "mode": "palette-classic"
2159               },
2160               "custom": {
2161                 "axisLabel": "Messages/s",
2162                 "axisPlacement": "auto",
2163                 "barAlignment": 0,
2164                 "drawStyle": "line",
2165                 "fillOpacity": 10,
2166                 "gradientMode": "none",
2167                 "hideFrom": {
2168                   "legend": false,
2169                   "tooltip": false,
2170                   "viz": false
2171                 },
2172                 "lineInterpolation": "linear",
2173                 "lineWidth": 2,
2174                 "pointSize": 5,
2175                 "scaleDistribution": {
2176                   "type": "linear"
2177                 },
2178                 "showPoints": "never",
2179                 "spanNulls": false,
2180                 "stacking": {
2181                   "group": "A",
2182                   "mode": "none"
2183                 },
2184                 "thresholdsStyle": {
2185                   "mode": "off"
2186                 }
2187               },
2188               "links": [],
2189               "mappings": [],
2190               "thresholds": {
2191                 "mode": "absolute",
2192                 "steps": [
2193                   {
2194                     "color": "green"
2195                   },
2196                   {
2197                     "color": "red",
2198                     "value": 80
2199                   }
2200                 ]
2201               },
2202               "unit": "cps"
2203             },
2204             "overrides": []
2205           },
2206           "gridPos": {
2207             "h": 8,
2208             "w": 8,
2209             "x": 0,
2210             "y": 32
2211           },
2212           "id": 10,
2213           "links": [],
2214           "options": {
2215             "legend": {
2216               "calcs": [
2217                 "mean",
2218                 "lastNotNull",
2219                 "max"
2220               ],
2221               "displayMode": "table",
2222               "placement": "bottom",
2223               "showLegend": true
2224             },
2225             "tooltip": {
2226               "mode": "single"
2227             }
2228           },
2229           "pluginVersion": "8.1.3",
2230           "targets": [
2231             {
2232               "datasource": {
2233                 "type": "prometheus",
2234                 "uid": "PBFA97CFB590B2093"
2235               },
2236               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_messagesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2237               "format": "time_series",
2238               "interval": "",
2239               "intervalFactor": 2,
2240               "legendFormat": "{{instance}}",
2241               "metric": "kafka_server_brokertopicmetrics_messagesinpersec",
2242               "refId": "A",
2243               "step": 4
2244             }
2245           ],
2246           "title": "Messages In Per Broker",
2247           "type": "timeseries"
2248         },
2249         {
2250           "datasource": {
2251             "type": "prometheus",
2252             "uid": "PBFA97CFB590B2093"
2253           },
2254           "fieldConfig": {
2255             "defaults": {
2256               "color": {
2257                 "mode": "palette-classic"
2258               },
2259               "custom": {
2260                 "axisLabel": "Bytes/s",
2261                 "axisPlacement": "auto",
2262                 "barAlignment": 0,
2263                 "drawStyle": "line",
2264                 "fillOpacity": 10,
2265                 "gradientMode": "none",
2266                 "hideFrom": {
2267                   "legend": false,
2268                   "tooltip": false,
2269                   "viz": false
2270                 },
2271                 "lineInterpolation": "linear",
2272                 "lineWidth": 2,
2273                 "pointSize": 5,
2274                 "scaleDistribution": {
2275                   "type": "linear"
2276                 },
2277                 "showPoints": "never",
2278                 "spanNulls": false,
2279                 "stacking": {
2280                   "group": "A",
2281                   "mode": "none"
2282                 },
2283                 "thresholdsStyle": {
2284                   "mode": "off"
2285                 }
2286               },
2287               "links": [],
2288               "mappings": [],
2289               "thresholds": {
2290                 "mode": "absolute",
2291                 "steps": [
2292                   {
2293                     "color": "green"
2294                   },
2295                   {
2296                     "color": "red",
2297                     "value": 80
2298                   }
2299                 ]
2300               },
2301               "unit": "Bps"
2302             },
2303             "overrides": []
2304           },
2305           "gridPos": {
2306             "h": 8,
2307             "w": 8,
2308             "x": 8,
2309             "y": 32
2310           },
2311           "id": 7,
2312           "links": [],
2313           "options": {
2314             "legend": {
2315               "calcs": [
2316                 "mean",
2317                 "lastNotNull",
2318                 "max"
2319               ],
2320               "displayMode": "table",
2321               "placement": "bottom",
2322               "showLegend": true
2323             },
2324             "tooltip": {
2325               "mode": "single"
2326             }
2327           },
2328           "pluginVersion": "8.1.3",
2329           "targets": [
2330             {
2331               "datasource": {
2332                 "type": "prometheus",
2333                 "uid": "PBFA97CFB590B2093"
2334               },
2335               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesinpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2336               "format": "time_series",
2337               "interval": "",
2338               "intervalFactor": 2,
2339               "legendFormat": "{{instance}}",
2340               "metric": "kafka_server_brokertopicmetrics_bytesinpersec",
2341               "refId": "A",
2342               "step": 4
2343             }
2344           ],
2345           "title": "Bytes In Per Broker",
2346           "type": "timeseries"
2347         },
2348         {
2349           "datasource": {
2350             "type": "prometheus",
2351             "uid": "PBFA97CFB590B2093"
2352           },
2353           "fieldConfig": {
2354             "defaults": {
2355               "color": {
2356                 "mode": "palette-classic"
2357               },
2358               "custom": {
2359                 "axisLabel": "",
2360                 "axisPlacement": "auto",
2361                 "barAlignment": 0,
2362                 "drawStyle": "line",
2363                 "fillOpacity": 10,
2364                 "gradientMode": "none",
2365                 "hideFrom": {
2366                   "legend": false,
2367                   "tooltip": false,
2368                   "viz": false
2369                 },
2370                 "lineInterpolation": "linear",
2371                 "lineWidth": 1,
2372                 "pointSize": 5,
2373                 "scaleDistribution": {
2374                   "type": "linear"
2375                 },
2376                 "showPoints": "never",
2377                 "spanNulls": true,
2378                 "stacking": {
2379                   "group": "A",
2380                   "mode": "none"
2381                 },
2382                 "thresholdsStyle": {
2383                   "mode": "off"
2384                 }
2385               },
2386               "links": [],
2387               "mappings": [],
2388               "min": 0,
2389               "thresholds": {
2390                 "mode": "absolute",
2391                 "steps": [
2392                   {
2393                     "color": "green"
2394                   },
2395                   {
2396                     "color": "red",
2397                     "value": 80
2398                   }
2399                 ]
2400               },
2401               "unit": "binBps"
2402             },
2403             "overrides": []
2404           },
2405           "gridPos": {
2406             "h": 8,
2407             "w": 8,
2408             "x": 16,
2409             "y": 32
2410           },
2411           "id": 9,
2412           "links": [],
2413           "options": {
2414             "legend": {
2415               "calcs": [
2416                 "mean",
2417                 "lastNotNull",
2418                 "max"
2419               ],
2420               "displayMode": "table",
2421               "placement": "bottom",
2422               "showLegend": true
2423             },
2424             "tooltip": {
2425               "mode": "single"
2426             }
2427           },
2428           "pluginVersion": "8.1.3",
2429           "targets": [
2430             {
2431               "datasource": {
2432                 "type": "prometheus",
2433                 "uid": "PBFA97CFB590B2093"
2434               },
2435               "expr": "sum without(topic)(rate(kafka_server_brokertopicmetrics_bytesoutpersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",topic!=\"\"}[5m]))",
2436               "format": "time_series",
2437               "interval": "",
2438               "intervalFactor": 1,
2439               "legendFormat": "{{instance}}",
2440               "refId": "A"
2441             }
2442           ],
2443           "title": "Bytes Out Per Broker",
2444           "type": "timeseries"
2445         }
2446       ],
2447       "targets": [
2448         {
2449           "datasource": {
2450             "type": "datasource",
2451             "uid": "grafana"
2452           },
2453           "refId": "A"
2454         }
2455       ],
2456       "title": "Throughput In/Out",
2457       "type": "row"
2458     },
2459     {
2460       "collapsed": true,
2461       "datasource": {
2462         "type": "datasource",
2463         "uid": "grafana"
2464       },
2465       "gridPos": {
2466         "h": 1,
2467         "w": 24,
2468         "x": 0,
2469         "y": 37
2470       },
2471       "id": 44,
2472       "panels": [
2473         {
2474           "datasource": {
2475             "type": "prometheus",
2476             "uid": "PBFA97CFB590B2093"
2477           },
2478           "description": "Average fraction of time the network processor threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available)\n",
2479           "fieldConfig": {
2480             "defaults": {
2481               "color": {
2482                 "mode": "palette-classic"
2483               },
2484               "custom": {
2485                 "axisLabel": "",
2486                 "axisPlacement": "auto",
2487                 "barAlignment": 0,
2488                 "drawStyle": "line",
2489                 "fillOpacity": 10,
2490                 "gradientMode": "none",
2491                 "hideFrom": {
2492                   "legend": false,
2493                   "tooltip": false,
2494                   "viz": false
2495                 },
2496                 "lineInterpolation": "linear",
2497                 "lineWidth": 1,
2498                 "pointSize": 5,
2499                 "scaleDistribution": {
2500                   "type": "linear"
2501                 },
2502                 "showPoints": "never",
2503                 "spanNulls": true,
2504                 "stacking": {
2505                   "group": "A",
2506                   "mode": "none"
2507                 },
2508                 "thresholdsStyle": {
2509                   "mode": "off"
2510                 }
2511               },
2512               "links": [],
2513               "mappings": [],
2514               "thresholds": {
2515                 "mode": "absolute",
2516                 "steps": [
2517                   {
2518                     "color": "green"
2519                   },
2520                   {
2521                     "color": "red",
2522                     "value": 80
2523                   }
2524                 ]
2525               },
2526               "unit": "percentunit"
2527             },
2528             "overrides": []
2529           },
2530           "gridPos": {
2531             "h": 9,
2532             "w": 12,
2533             "x": 0,
2534             "y": 25
2535           },
2536           "id": 24,
2537           "options": {
2538             "legend": {
2539               "calcs": [
2540                 "mean",
2541                 "lastNotNull",
2542                 "max"
2543               ],
2544               "displayMode": "table",
2545               "placement": "bottom",
2546               "showLegend": true
2547             },
2548             "tooltip": {
2549               "mode": "single"
2550             }
2551           },
2552           "pluginVersion": "8.1.3",
2553           "targets": [
2554             {
2555               "datasource": {
2556                 "type": "prometheus",
2557                 "uid": "PBFA97CFB590B2093"
2558               },
2559               "expr": "1-kafka_network_socketserver_networkprocessoravgidlepercent{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2560               "interval": "",
2561               "legendFormat": "{{instance}}",
2562               "refId": "A"
2563             }
2564           ],
2565           "title": "Network Processor Avg Usage Percent",
2566           "type": "timeseries"
2567         },
2568         {
2569           "datasource": {
2570             "type": "prometheus",
2571             "uid": "PBFA97CFB590B2093"
2572           },
2573           "description": "Average fraction of time the request handler threads are idle. Values are between 0 (all resources are used) and 100 (all resources are available).\n",
2574           "fieldConfig": {
2575             "defaults": {
2576               "color": {
2577                 "mode": "palette-classic"
2578               },
2579               "custom": {
2580                 "axisLabel": "",
2581                 "axisPlacement": "auto",
2582                 "barAlignment": 0,
2583                 "drawStyle": "line",
2584                 "fillOpacity": 10,
2585                 "gradientMode": "none",
2586                 "hideFrom": {
2587                   "legend": false,
2588                   "tooltip": false,
2589                   "viz": false
2590                 },
2591                 "lineInterpolation": "linear",
2592                 "lineWidth": 1,
2593                 "pointSize": 5,
2594                 "scaleDistribution": {
2595                   "type": "linear"
2596                 },
2597                 "showPoints": "never",
2598                 "spanNulls": true,
2599                 "stacking": {
2600                   "group": "A",
2601                   "mode": "none"
2602                 },
2603                 "thresholdsStyle": {
2604                   "mode": "off"
2605                 }
2606               },
2607               "links": [],
2608               "mappings": [],
2609               "min": 0,
2610               "thresholds": {
2611                 "mode": "absolute",
2612                 "steps": [
2613                   {
2614                     "color": "green"
2615                   },
2616                   {
2617                     "color": "red",
2618                     "value": 80
2619                   }
2620                 ]
2621               },
2622               "unit": "percentunit"
2623             },
2624             "overrides": []
2625           },
2626           "gridPos": {
2627             "h": 9,
2628             "w": 12,
2629             "x": 12,
2630             "y": 25
2631           },
2632           "id": 25,
2633           "options": {
2634             "legend": {
2635               "calcs": [
2636                 "mean",
2637                 "lastNotNull",
2638                 "max"
2639               ],
2640               "displayMode": "table",
2641               "placement": "bottom",
2642               "showLegend": true
2643             },
2644             "tooltip": {
2645               "mode": "single"
2646             }
2647           },
2648           "pluginVersion": "8.1.3",
2649           "targets": [
2650             {
2651               "datasource": {
2652                 "type": "prometheus",
2653                 "uid": "PBFA97CFB590B2093"
2654               },
2655               "expr": "1 - kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2656               "interval": "",
2657               "legendFormat": "{{instance}}",
2658               "refId": "A"
2659             }
2660           ],
2661           "title": "Request Handler Avg Percent",
2662           "type": "timeseries"
2663         }
2664       ],
2665       "targets": [
2666         {
2667           "datasource": {
2668             "type": "datasource",
2669             "uid": "grafana"
2670           },
2671           "refId": "A"
2672         }
2673       ],
2674       "title": "Thread utilization",
2675       "type": "row"
2676     },
2677     {
2678       "collapsed": true,
2679       "datasource": {
2680         "type": "datasource",
2681         "uid": "grafana"
2682       },
2683       "gridPos": {
2684         "h": 1,
2685         "w": 24,
2686         "x": 0,
2687         "y": 38
2688       },
2689       "id": 86,
2690       "panels": [
2691         {
2692           "datasource": {
2693             "type": "prometheus",
2694             "uid": "PBFA97CFB590B2093"
2695           },
2696           "description": "Latency in millseconds for ZooKeeper requests from broker.\n",
2697           "fieldConfig": {
2698             "defaults": {
2699               "color": {
2700                 "mode": "palette-classic"
2701               },
2702               "custom": {
2703                 "axisLabel": "",
2704                 "axisPlacement": "auto",
2705                 "barAlignment": 0,
2706                 "drawStyle": "line",
2707                 "fillOpacity": 10,
2708                 "gradientMode": "none",
2709                 "hideFrom": {
2710                   "legend": false,
2711                   "tooltip": false,
2712                   "viz": false
2713                 },
2714                 "lineInterpolation": "linear",
2715                 "lineWidth": 1,
2716                 "pointSize": 5,
2717                 "scaleDistribution": {
2718                   "type": "linear"
2719                 },
2720                 "showPoints": "never",
2721                 "spanNulls": true,
2722                 "stacking": {
2723                   "group": "A",
2724                   "mode": "none"
2725                 },
2726                 "thresholdsStyle": {
2727                   "mode": "off"
2728                 }
2729               },
2730               "links": [],
2731               "mappings": [],
2732               "thresholds": {
2733                 "mode": "absolute",
2734                 "steps": [
2735                   {
2736                     "color": "green"
2737                   },
2738                   {
2739                     "color": "red",
2740                     "value": 80
2741                   }
2742                 ]
2743               },
2744               "unit": "ms"
2745             },
2746             "overrides": []
2747           },
2748           "gridPos": {
2749             "h": 9,
2750             "w": 12,
2751             "x": 0,
2752             "y": 26
2753           },
2754           "id": 88,
2755           "options": {
2756             "legend": {
2757               "calcs": [
2758                 "mean",
2759                 "lastNotNull",
2760                 "max"
2761               ],
2762               "displayMode": "table",
2763               "placement": "bottom",
2764               "showLegend": true
2765             },
2766             "tooltip": {
2767               "mode": "single"
2768             }
2769           },
2770           "pluginVersion": "8.1.3",
2771           "targets": [
2772             {
2773               "datasource": {
2774                 "type": "prometheus",
2775                 "uid": "PBFA97CFB590B2093"
2776               },
2777               "expr": "kafka_server_zookeeperclientmetrics_zookeeperrequestlatencyms{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\"}",
2778               "hide": false,
2779               "legendFormat": "{{instance}} - {{quantile}}",
2780               "refId": "A"
2781             }
2782           ],
2783           "title": "Zookeeper Request Latency",
2784           "type": "timeseries"
2785         },
2786         {
2787           "datasource": {
2788             "type": "prometheus",
2789             "uid": "PBFA97CFB590B2093"
2790           },
2791           "description": "",
2792           "fieldConfig": {
2793             "defaults": {
2794               "color": {
2795                 "mode": "palette-classic"
2796               },
2797               "custom": {
2798                 "axisLabel": "",
2799                 "axisPlacement": "auto",
2800                 "barAlignment": 0,
2801                 "drawStyle": "line",
2802                 "fillOpacity": 10,
2803                 "gradientMode": "none",
2804                 "hideFrom": {
2805                   "legend": false,
2806                   "tooltip": false,
2807                   "viz": false
2808                 },
2809                 "lineInterpolation": "linear",
2810                 "lineWidth": 1,
2811                 "pointSize": 5,
2812                 "scaleDistribution": {
2813                   "type": "linear"
2814                 },
2815                 "showPoints": "never",
2816                 "spanNulls": true,
2817                 "stacking": {
2818                   "group": "A",
2819                   "mode": "none"
2820                 },
2821                 "thresholdsStyle": {
2822                   "mode": "off"
2823                 }
2824               },
2825               "links": [],
2826               "mappings": [],
2827               "min": 0,
2828               "thresholds": {
2829                 "mode": "absolute",
2830                 "steps": [
2831                   {
2832                     "color": "green"
2833                   },
2834                   {
2835                     "color": "red",
2836                     "value": 80
2837                   }
2838                 ]
2839               },
2840               "unit": "short"
2841             },
2842             "overrides": []
2843           },
2844           "gridPos": {
2845             "h": 9,
2846             "w": 12,
2847             "x": 12,
2848             "y": 26
2849           },
2850           "id": 92,
2851           "options": {
2852             "legend": {
2853               "calcs": [
2854                 "mean",
2855                 "lastNotNull",
2856                 "max"
2857               ],
2858               "displayMode": "table",
2859               "placement": "bottom",
2860               "showLegend": true
2861             },
2862             "tooltip": {
2863               "mode": "single"
2864             }
2865           },
2866           "pluginVersion": "8.1.3",
2867           "targets": [
2868             {
2869               "datasource": {
2870                 "type": "prometheus",
2871                 "uid": "PBFA97CFB590B2093"
2872               },
2873               "expr": "kafka_server_sessionexpirelistener_zookeepersyncconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2874               "hide": false,
2875               "instant": false,
2876               "interval": "",
2877               "legendFormat": "{{instance}}",
2878               "refId": "A"
2879             }
2880           ],
2881           "title": "Zookeeper connections per sec",
2882           "type": "timeseries"
2883         },
2884         {
2885           "datasource": {
2886             "type": "prometheus",
2887             "uid": "PBFA97CFB590B2093"
2888           },
2889           "description": "",
2890           "fieldConfig": {
2891             "defaults": {
2892               "color": {
2893                 "mode": "palette-classic"
2894               },
2895               "custom": {
2896                 "axisLabel": "",
2897                 "axisPlacement": "auto",
2898                 "barAlignment": 0,
2899                 "drawStyle": "line",
2900                 "fillOpacity": 10,
2901                 "gradientMode": "none",
2902                 "hideFrom": {
2903                   "legend": false,
2904                   "tooltip": false,
2905                   "viz": false
2906                 },
2907                 "lineInterpolation": "linear",
2908                 "lineWidth": 1,
2909                 "pointSize": 5,
2910                 "scaleDistribution": {
2911                   "type": "linear"
2912                 },
2913                 "showPoints": "never",
2914                 "spanNulls": true,
2915                 "stacking": {
2916                   "group": "A",
2917                   "mode": "none"
2918                 },
2919                 "thresholdsStyle": {
2920                   "mode": "off"
2921                 }
2922               },
2923               "links": [],
2924               "mappings": [],
2925               "min": 0,
2926               "thresholds": {
2927                 "mode": "absolute",
2928                 "steps": [
2929                   {
2930                     "color": "green"
2931                   },
2932                   {
2933                     "color": "red",
2934                     "value": 80
2935                   }
2936                 ]
2937               },
2938               "unit": "short"
2939             },
2940             "overrides": []
2941           },
2942           "gridPos": {
2943             "h": 9,
2944             "w": 8,
2945             "x": 0,
2946             "y": 35
2947           },
2948           "id": 89,
2949           "options": {
2950             "legend": {
2951               "calcs": [
2952                 "mean",
2953                 "lastNotNull",
2954                 "max"
2955               ],
2956               "displayMode": "table",
2957               "placement": "bottom",
2958               "showLegend": true
2959             },
2960             "tooltip": {
2961               "mode": "single"
2962             }
2963           },
2964           "pluginVersion": "8.1.3",
2965           "targets": [
2966             {
2967               "datasource": {
2968                 "type": "prometheus",
2969                 "uid": "PBFA97CFB590B2093"
2970               },
2971               "expr": "kafka_server_sessionexpirelistener_zookeeperexpirespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
2972               "hide": false,
2973               "interval": "",
2974               "legendFormat": "{{instance}}",
2975               "refId": "A"
2976             }
2977           ],
2978           "title": "Zookeeper expired connections per sec",
2979           "type": "timeseries"
2980         },
2981         {
2982           "datasource": {
2983             "type": "prometheus",
2984             "uid": "PBFA97CFB590B2093"
2985           },
2986           "description": "",
2987           "fieldConfig": {
2988             "defaults": {
2989               "color": {
2990                 "mode": "palette-classic"
2991               },
2992               "custom": {
2993                 "axisLabel": "",
2994                 "axisPlacement": "auto",
2995                 "barAlignment": 0,
2996                 "drawStyle": "line",
2997                 "fillOpacity": 10,
2998                 "gradientMode": "none",
2999                 "hideFrom": {
3000                   "legend": false,
3001                   "tooltip": false,
3002                   "viz": false
3003                 },
3004                 "lineInterpolation": "linear",
3005                 "lineWidth": 1,
3006                 "pointSize": 5,
3007                 "scaleDistribution": {
3008                   "type": "linear"
3009                 },
3010                 "showPoints": "never",
3011                 "spanNulls": true,
3012                 "stacking": {
3013                   "group": "A",
3014                   "mode": "none"
3015                 },
3016                 "thresholdsStyle": {
3017                   "mode": "off"
3018                 }
3019               },
3020               "links": [],
3021               "mappings": [],
3022               "min": 0,
3023               "thresholds": {
3024                 "mode": "absolute",
3025                 "steps": [
3026                   {
3027                     "color": "green"
3028                   },
3029                   {
3030                     "color": "red",
3031                     "value": 80
3032                   }
3033                 ]
3034               },
3035               "unit": "short"
3036             },
3037             "overrides": []
3038           },
3039           "gridPos": {
3040             "h": 9,
3041             "w": 8,
3042             "x": 8,
3043             "y": 35
3044           },
3045           "id": 90,
3046           "options": {
3047             "legend": {
3048               "calcs": [
3049                 "mean",
3050                 "lastNotNull",
3051                 "max"
3052               ],
3053               "displayMode": "table",
3054               "placement": "bottom",
3055               "showLegend": true
3056             },
3057             "tooltip": {
3058               "mode": "single"
3059             }
3060           },
3061           "pluginVersion": "8.1.3",
3062           "targets": [
3063             {
3064               "datasource": {
3065                 "type": "prometheus",
3066                 "uid": "PBFA97CFB590B2093"
3067               },
3068               "expr": "kafka_server_sessionexpirelistener_zookeeperdisconnectspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
3069               "hide": false,
3070               "interval": "",
3071               "legendFormat": "{{instance}}",
3072               "refId": "A"
3073             }
3074           ],
3075           "title": "Zookeeper disconnect per sec",
3076           "type": "timeseries"
3077         },
3078         {
3079           "datasource": {
3080             "type": "prometheus",
3081             "uid": "PBFA97CFB590B2093"
3082           },
3083           "description": "",
3084           "fieldConfig": {
3085             "defaults": {
3086               "color": {
3087                 "mode": "palette-classic"
3088               },
3089               "custom": {
3090                 "axisLabel": "",
3091                 "axisPlacement": "auto",
3092                 "barAlignment": 0,
3093                 "drawStyle": "line",
3094                 "fillOpacity": 10,
3095                 "gradientMode": "none",
3096                 "hideFrom": {
3097                   "legend": false,
3098                   "tooltip": false,
3099                   "viz": false
3100                 },
3101                 "lineInterpolation": "linear",
3102                 "lineWidth": 1,
3103                 "pointSize": 5,
3104                 "scaleDistribution": {
3105                   "type": "linear"
3106                 },
3107                 "showPoints": "never",
3108                 "spanNulls": true,
3109                 "stacking": {
3110                   "group": "A",
3111                   "mode": "none"
3112                 },
3113                 "thresholdsStyle": {
3114                   "mode": "off"
3115                 }
3116               },
3117               "links": [],
3118               "mappings": [],
3119               "min": 0,
3120               "thresholds": {
3121                 "mode": "absolute",
3122                 "steps": [
3123                   {
3124                     "color": "green"
3125                   },
3126                   {
3127                     "color": "red",
3128                     "value": 80
3129                   }
3130                 ]
3131               },
3132               "unit": "short"
3133             },
3134             "overrides": []
3135           },
3136           "gridPos": {
3137             "h": 9,
3138             "w": 8,
3139             "x": 16,
3140             "y": 35
3141           },
3142           "id": 91,
3143           "options": {
3144             "legend": {
3145               "calcs": [
3146                 "mean",
3147                 "lastNotNull",
3148                 "max"
3149               ],
3150               "displayMode": "table",
3151               "placement": "bottom",
3152               "showLegend": true
3153             },
3154             "tooltip": {
3155               "mode": "single"
3156             }
3157           },
3158           "pluginVersion": "8.1.3",
3159           "targets": [
3160             {
3161               "datasource": {
3162                 "type": "prometheus",
3163                 "uid": "PBFA97CFB590B2093"
3164               },
3165               "expr": "kafka_server_sessionexpirelistener_zookeeperauthfailurespersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}",
3166               "hide": false,
3167               "interval": "",
3168               "legendFormat": "{{instance}}",
3169               "refId": "A"
3170             }
3171           ],
3172           "title": "Zookeeper auth failures per sec",
3173           "type": "timeseries"
3174         }
3175       ],
3176       "targets": [
3177         {
3178           "datasource": {
3179             "type": "datasource",
3180             "uid": "grafana"
3181           },
3182           "refId": "A"
3183         }
3184       ],
3185       "title": "Zookeeper",
3186       "type": "row"
3187     },
3188     {
3189       "collapsed": false,
3190       "datasource": {
3191         "type": "datasource",
3192         "uid": "grafana"
3193       },
3194       "gridPos": {
3195         "h": 1,
3196         "w": 24,
3197         "x": 0,
3198         "y": 39
3199       },
3200       "id": 82,
3201       "panels": [],
3202       "targets": [
3203         {
3204           "datasource": {
3205             "type": "datasource",
3206             "uid": "grafana"
3207           },
3208           "refId": "A"
3209         }
3210       ],
3211       "title": "Isr Shrinks / Expands",
3212       "type": "row"
3213     },
3214     {
3215       "datasource": {
3216         "type": "prometheus",
3217         "uid": "PBFA97CFB590B2093"
3218       },
3219       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
3220       "fieldConfig": {
3221         "defaults": {
3222           "color": {
3223             "mode": "palette-classic"
3224           },
3225           "custom": {
3226             "axisCenteredZero": false,
3227             "axisColorMode": "text",
3228             "axisLabel": "",
3229             "axisPlacement": "auto",
3230             "barAlignment": 0,
3231             "drawStyle": "line",
3232             "fillOpacity": 10,
3233             "gradientMode": "none",
3234             "hideFrom": {
3235               "legend": false,
3236               "tooltip": false,
3237               "viz": false
3238             },
3239             "lineInterpolation": "linear",
3240             "lineWidth": 1,
3241             "pointSize": 5,
3242             "scaleDistribution": {
3243               "type": "linear"
3244             },
3245             "showPoints": "never",
3246             "spanNulls": true,
3247             "stacking": {
3248               "group": "A",
3249               "mode": "none"
3250             },
3251             "thresholdsStyle": {
3252               "mode": "off"
3253             }
3254           },
3255           "links": [],
3256           "mappings": [],
3257           "min": 0,
3258           "thresholds": {
3259             "mode": "absolute",
3260             "steps": [
3261               {
3262                 "color": "green",
3263                 "value": null
3264               },
3265               {
3266                 "color": "red",
3267                 "value": 80
3268               }
3269             ]
3270           },
3271           "unit": "ops"
3272         },
3273         "overrides": []
3274       },
3275       "gridPos": {
3276         "h": 9,
3277         "w": 12,
3278         "x": 0,
3279         "y": 40
3280       },
3281       "id": 80,
3282       "options": {
3283         "legend": {
3284           "calcs": [
3285             "mean",
3286             "lastNotNull",
3287             "max"
3288           ],
3289           "displayMode": "table",
3290           "placement": "bottom",
3291           "showLegend": true
3292         },
3293         "tooltip": {
3294           "mode": "single",
3295           "sort": "none"
3296         }
3297       },
3298       "pluginVersion": "8.1.3",
3299       "targets": [
3300         {
3301           "datasource": {
3302             "type": "prometheus",
3303             "uid": "PBFA97CFB590B2093"
3304           },
3305           "expr": "rate(kafka_server_replicamanager_isrshrinkspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3306           "interval": "",
3307           "legendFormat": "{{instance}}",
3308           "refId": "A"
3309         }
3310       ],
3311       "title": "IsrShrinks per Sec",
3312       "type": "timeseries"
3313     },
3314     {
3315       "datasource": {
3316         "type": "prometheus",
3317         "uid": "PBFA97CFB590B2093"
3318       },
3319       "description": ": The number of in-sync replicas (ISRs) for a particular partition should remain fairly static, the only exceptions are when you are expanding your broker cluster or removing partitions. In order to maintain high availability, a healthy Kafka cluster requires a minimum number of ISRs for failover. A replica could be removed from the ISR pool for a couple of reasons: it is too far behind the leaders offset (user-configurable by setting the replica.lag.max.messages configuration parameter), or it has not contacted the leader for some time (configurable with the replica.socket.timeout.ms parameter). No matter the reason, an increase in IsrShrinksPerSec without a corresponding increase in IsrExpandsPerSec shortly thereafter is cause for concern and requires user intervention.The Kafka documentation provides a wealth of information on the user-configurable parameters for brokers.",
3320       "fieldConfig": {
3321         "defaults": {
3322           "color": {
3323             "mode": "palette-classic"
3324           },
3325           "custom": {
3326             "axisCenteredZero": false,
3327             "axisColorMode": "text",
3328             "axisLabel": "",
3329             "axisPlacement": "auto",
3330             "barAlignment": 0,
3331             "drawStyle": "line",
3332             "fillOpacity": 10,
3333             "gradientMode": "none",
3334             "hideFrom": {
3335               "legend": false,
3336               "tooltip": false,
3337               "viz": false
3338             },
3339             "lineInterpolation": "linear",
3340             "lineWidth": 1,
3341             "pointSize": 5,
3342             "scaleDistribution": {
3343               "type": "linear"
3344             },
3345             "showPoints": "never",
3346             "spanNulls": true,
3347             "stacking": {
3348               "group": "A",
3349               "mode": "none"
3350             },
3351             "thresholdsStyle": {
3352               "mode": "off"
3353             }
3354           },
3355           "links": [],
3356           "mappings": [],
3357           "min": 0,
3358           "thresholds": {
3359             "mode": "absolute",
3360             "steps": [
3361               {
3362                 "color": "green",
3363                 "value": null
3364               },
3365               {
3366                 "color": "red",
3367                 "value": 80
3368               }
3369             ]
3370           },
3371           "unit": "ops"
3372         },
3373         "overrides": []
3374       },
3375       "gridPos": {
3376         "h": 9,
3377         "w": 12,
3378         "x": 12,
3379         "y": 40
3380       },
3381       "id": 83,
3382       "options": {
3383         "legend": {
3384           "calcs": [
3385             "mean",
3386             "lastNotNull",
3387             "max"
3388           ],
3389           "displayMode": "table",
3390           "placement": "bottom",
3391           "showLegend": true
3392         },
3393         "tooltip": {
3394           "mode": "single",
3395           "sort": "none"
3396         }
3397       },
3398       "pluginVersion": "8.1.3",
3399       "targets": [
3400         {
3401           "datasource": {
3402             "type": "prometheus",
3403             "uid": "PBFA97CFB590B2093"
3404           },
3405           "expr": "rate(kafka_server_replicamanager_isrexpandspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}[5m])",
3406           "hide": false,
3407           "interval": "",
3408           "legendFormat": "{{instance}}",
3409           "refId": "A"
3410         }
3411       ],
3412       "title": "IsrExpands per Sec",
3413       "type": "timeseries"
3414     },
3415     {
3416       "collapsed": true,
3417       "datasource": {
3418         "type": "datasource",
3419         "uid": "grafana"
3420       },
3421       "gridPos": {
3422         "h": 1,
3423         "w": 24,
3424         "x": 0,
3425         "y": 49
3426       },
3427       "id": 53,
3428       "panels": [
3429         {
3430           "datasource": {
3431             "type": "prometheus",
3432             "uid": "PBFA97CFB590B2093"
3433           },
3434           "fieldConfig": {
3435             "defaults": {
3436               "color": {
3437                 "mode": "palette-classic"
3438               },
3439               "custom": {
3440                 "axisLabel": "",
3441                 "axisPlacement": "auto",
3442                 "barAlignment": 0,
3443                 "drawStyle": "line",
3444                 "fillOpacity": 10,
3445                 "gradientMode": "none",
3446                 "hideFrom": {
3447                   "legend": false,
3448                   "tooltip": false,
3449                   "viz": false
3450                 },
3451                 "lineInterpolation": "linear",
3452                 "lineWidth": 1,
3453                 "pointSize": 5,
3454                 "scaleDistribution": {
3455                   "type": "linear"
3456                 },
3457                 "showPoints": "never",
3458                 "spanNulls": true,
3459                 "stacking": {
3460                   "group": "A",
3461                   "mode": "none"
3462                 },
3463                 "thresholdsStyle": {
3464                   "mode": "off"
3465                 }
3466               },
3467               "links": [],
3468               "mappings": [],
3469               "thresholds": {
3470                 "mode": "absolute",
3471                 "steps": [
3472                   {
3473                     "color": "green"
3474                   },
3475                   {
3476                     "color": "red",
3477                     "value": 80
3478                   }
3479                 ]
3480               },
3481               "unit": "bytes"
3482             },
3483             "overrides": []
3484           },
3485           "gridPos": {
3486             "h": 9,
3487             "w": 12,
3488             "x": 0,
3489             "y": 28
3490           },
3491           "id": 55,
3492           "options": {
3493             "legend": {
3494               "calcs": [
3495                 "mean",
3496                 "lastNotNull",
3497                 "max"
3498               ],
3499               "displayMode": "table",
3500               "placement": "bottom",
3501               "showLegend": true
3502             },
3503             "tooltip": {
3504               "mode": "single"
3505             }
3506           },
3507           "pluginVersion": "8.1.3",
3508           "targets": [
3509             {
3510               "datasource": {
3511                 "type": "prometheus",
3512                 "uid": "PBFA97CFB590B2093"
3513               },
3514               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (topic)",
3515               "legendFormat": "{{topic}}",
3516               "refId": "A"
3517             }
3518           ],
3519           "title": "Log size per Topic",
3520           "type": "timeseries"
3521         },
3522         {
3523           "datasource": {
3524             "type": "prometheus",
3525             "uid": "PBFA97CFB590B2093"
3526           },
3527           "fieldConfig": {
3528             "defaults": {
3529               "color": {
3530                 "mode": "palette-classic"
3531               },
3532               "custom": {
3533                 "axisLabel": "",
3534                 "axisPlacement": "auto",
3535                 "barAlignment": 0,
3536                 "drawStyle": "line",
3537                 "fillOpacity": 10,
3538                 "gradientMode": "none",
3539                 "hideFrom": {
3540                   "legend": false,
3541                   "tooltip": false,
3542                   "viz": false
3543                 },
3544                 "lineInterpolation": "linear",
3545                 "lineWidth": 1,
3546                 "pointSize": 5,
3547                 "scaleDistribution": {
3548                   "type": "linear"
3549                 },
3550                 "showPoints": "never",
3551                 "spanNulls": true,
3552                 "stacking": {
3553                   "group": "A",
3554                   "mode": "none"
3555                 },
3556                 "thresholdsStyle": {
3557                   "mode": "off"
3558                 }
3559               },
3560               "links": [],
3561               "mappings": [],
3562               "thresholds": {
3563                 "mode": "absolute",
3564                 "steps": [
3565                   {
3566                     "color": "green"
3567                   },
3568                   {
3569                     "color": "red",
3570                     "value": 80
3571                   }
3572                 ]
3573               },
3574               "unit": "bytes"
3575             },
3576             "overrides": []
3577           },
3578           "gridPos": {
3579             "h": 9,
3580             "w": 12,
3581             "x": 12,
3582             "y": 28
3583           },
3584           "id": 56,
3585           "options": {
3586             "legend": {
3587               "calcs": [
3588                 "mean",
3589                 "lastNotNull",
3590                 "max"
3591               ],
3592               "displayMode": "table",
3593               "placement": "bottom",
3594               "showLegend": true
3595             },
3596             "tooltip": {
3597               "mode": "single"
3598             }
3599           },
3600           "pluginVersion": "8.1.3",
3601           "targets": [
3602             {
3603               "datasource": {
3604                 "type": "prometheus",
3605                 "uid": "PBFA97CFB590B2093"
3606               },
3607               "expr": "sum(kafka_log_log_size{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"}) by (instance)",
3608               "legendFormat": "{{instance}}",
3609               "refId": "A"
3610             }
3611           ],
3612           "title": "Log size per Broker",
3613           "type": "timeseries"
3614         }
3615       ],
3616       "targets": [
3617         {
3618           "datasource": {
3619             "type": "datasource",
3620             "uid": "grafana"
3621           },
3622           "refId": "A"
3623         }
3624       ],
3625       "title": "Logs size",
3626       "type": "row"
3627     },
3628     {
3629       "collapsed": true,
3630       "datasource": {
3631         "type": "datasource",
3632         "uid": "grafana"
3633       },
3634       "gridPos": {
3635         "h": 1,
3636         "w": 24,
3637         "x": 0,
3638         "y": 50
3639       },
3640       "id": 58,
3641       "panels": [
3642         {
3643           "datasource": {
3644             "type": "prometheus",
3645             "uid": "PBFA97CFB590B2093"
3646           },
3647           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
3648           "fieldConfig": {
3649             "defaults": {
3650               "color": {
3651                 "mode": "palette-classic"
3652               },
3653               "custom": {
3654                 "axisLabel": "",
3655                 "axisPlacement": "auto",
3656                 "barAlignment": 0,
3657                 "drawStyle": "line",
3658                 "fillOpacity": 10,
3659                 "gradientMode": "none",
3660                 "hideFrom": {
3661                   "legend": false,
3662                   "tooltip": false,
3663                   "viz": false
3664                 },
3665                 "lineInterpolation": "linear",
3666                 "lineWidth": 1,
3667                 "pointSize": 5,
3668                 "scaleDistribution": {
3669                   "type": "linear"
3670                 },
3671                 "showPoints": "never",
3672                 "spanNulls": true,
3673                 "stacking": {
3674                   "group": "A",
3675                   "mode": "none"
3676                 },
3677                 "thresholdsStyle": {
3678                   "mode": "off"
3679                 }
3680               },
3681               "links": [],
3682               "mappings": [],
3683               "thresholds": {
3684                 "mode": "absolute",
3685                 "steps": [
3686                   {
3687                     "color": "green"
3688                   },
3689                   {
3690                     "color": "red",
3691                     "value": 80
3692                   }
3693                 ]
3694               },
3695               "unit": "ms"
3696             },
3697             "overrides": []
3698           },
3699           "gridPos": {
3700             "h": 9,
3701             "w": 12,
3702             "x": 0,
3703             "y": 29
3704           },
3705           "id": 60,
3706           "options": {
3707             "legend": {
3708               "calcs": [
3709                 "mean",
3710                 "lastNotNull",
3711                 "max"
3712               ],
3713               "displayMode": "list",
3714               "placement": "bottom",
3715               "showLegend": true
3716             },
3717             "tooltip": {
3718               "mode": "single"
3719             }
3720           },
3721           "pluginVersion": "8.1.3",
3722           "targets": [
3723             {
3724               "datasource": {
3725                 "type": "prometheus",
3726                 "uid": "PBFA97CFB590B2093"
3727               },
3728               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3729               "hide": false,
3730               "legendFormat": "{{instance}} - {{quantile}}",
3731               "refId": "A"
3732             }
3733           ],
3734           "title": "Producer - RequestQueueTimeMs",
3735           "type": "timeseries"
3736         },
3737         {
3738           "datasource": {
3739             "type": "prometheus",
3740             "uid": "PBFA97CFB590B2093"
3741           },
3742           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
3743           "fieldConfig": {
3744             "defaults": {
3745               "color": {
3746                 "mode": "palette-classic"
3747               },
3748               "custom": {
3749                 "axisLabel": "",
3750                 "axisPlacement": "auto",
3751                 "barAlignment": 0,
3752                 "drawStyle": "line",
3753                 "fillOpacity": 10,
3754                 "gradientMode": "none",
3755                 "hideFrom": {
3756                   "legend": false,
3757                   "tooltip": false,
3758                   "viz": false
3759                 },
3760                 "lineInterpolation": "linear",
3761                 "lineWidth": 1,
3762                 "pointSize": 5,
3763                 "scaleDistribution": {
3764                   "type": "linear"
3765                 },
3766                 "showPoints": "never",
3767                 "spanNulls": true,
3768                 "stacking": {
3769                   "group": "A",
3770                   "mode": "none"
3771                 },
3772                 "thresholdsStyle": {
3773                   "mode": "off"
3774                 }
3775               },
3776               "links": [],
3777               "mappings": [],
3778               "thresholds": {
3779                 "mode": "absolute",
3780                 "steps": [
3781                   {
3782                     "color": "green"
3783                   },
3784                   {
3785                     "color": "red",
3786                     "value": 80
3787                   }
3788                 ]
3789               },
3790               "unit": "ms"
3791             },
3792             "overrides": []
3793           },
3794           "gridPos": {
3795             "h": 9,
3796             "w": 12,
3797             "x": 12,
3798             "y": 29
3799           },
3800           "id": 61,
3801           "options": {
3802             "legend": {
3803               "calcs": [
3804                 "mean",
3805                 "lastNotNull",
3806                 "max"
3807               ],
3808               "displayMode": "list",
3809               "placement": "bottom",
3810               "showLegend": true
3811             },
3812             "tooltip": {
3813               "mode": "single"
3814             }
3815           },
3816           "pluginVersion": "8.1.3",
3817           "targets": [
3818             {
3819               "datasource": {
3820                 "type": "prometheus",
3821                 "uid": "PBFA97CFB590B2093"
3822               },
3823               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3824               "hide": false,
3825               "legendFormat": "{{instance}} - {{quantile}}",
3826               "refId": "A"
3827             }
3828           ],
3829           "title": "Producer - LocalTimeMs",
3830           "type": "timeseries"
3831         },
3832         {
3833           "datasource": {
3834             "type": "prometheus",
3835             "uid": "PBFA97CFB590B2093"
3836           },
3837           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
3838           "fieldConfig": {
3839             "defaults": {
3840               "color": {
3841                 "mode": "palette-classic"
3842               },
3843               "custom": {
3844                 "axisLabel": "",
3845                 "axisPlacement": "auto",
3846                 "barAlignment": 0,
3847                 "drawStyle": "line",
3848                 "fillOpacity": 10,
3849                 "gradientMode": "none",
3850                 "hideFrom": {
3851                   "legend": false,
3852                   "tooltip": false,
3853                   "viz": false
3854                 },
3855                 "lineInterpolation": "linear",
3856                 "lineWidth": 1,
3857                 "pointSize": 5,
3858                 "scaleDistribution": {
3859                   "type": "linear"
3860                 },
3861                 "showPoints": "never",
3862                 "spanNulls": true,
3863                 "stacking": {
3864                   "group": "A",
3865                   "mode": "none"
3866                 },
3867                 "thresholdsStyle": {
3868                   "mode": "off"
3869                 }
3870               },
3871               "links": [],
3872               "mappings": [],
3873               "thresholds": {
3874                 "mode": "absolute",
3875                 "steps": [
3876                   {
3877                     "color": "green"
3878                   },
3879                   {
3880                     "color": "red",
3881                     "value": 80
3882                   }
3883                 ]
3884               },
3885               "unit": "ms"
3886             },
3887             "overrides": []
3888           },
3889           "gridPos": {
3890             "h": 10,
3891             "w": 8,
3892             "x": 0,
3893             "y": 38
3894           },
3895           "id": 62,
3896           "options": {
3897             "legend": {
3898               "calcs": [
3899                 "mean",
3900                 "lastNotNull",
3901                 "max"
3902               ],
3903               "displayMode": "table",
3904               "placement": "bottom",
3905               "showLegend": true
3906             },
3907             "tooltip": {
3908               "mode": "single"
3909             }
3910           },
3911           "pluginVersion": "8.1.3",
3912           "targets": [
3913             {
3914               "datasource": {
3915                 "type": "prometheus",
3916                 "uid": "PBFA97CFB590B2093"
3917               },
3918               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
3919               "hide": false,
3920               "legendFormat": "{{instance}} - {{quantile}}",
3921               "refId": "A"
3922             }
3923           ],
3924           "title": "Producer - RemoteTimeMs",
3925           "type": "timeseries"
3926         },
3927         {
3928           "datasource": {
3929             "type": "prometheus",
3930             "uid": "PBFA97CFB590B2093"
3931           },
3932           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
3933           "fieldConfig": {
3934             "defaults": {
3935               "color": {
3936                 "mode": "palette-classic"
3937               },
3938               "custom": {
3939                 "axisLabel": "",
3940                 "axisPlacement": "auto",
3941                 "barAlignment": 0,
3942                 "drawStyle": "line",
3943                 "fillOpacity": 10,
3944                 "gradientMode": "none",
3945                 "hideFrom": {
3946                   "legend": false,
3947                   "tooltip": false,
3948                   "viz": false
3949                 },
3950                 "lineInterpolation": "linear",
3951                 "lineWidth": 1,
3952                 "pointSize": 5,
3953                 "scaleDistribution": {
3954                   "type": "linear"
3955                 },
3956                 "showPoints": "never",
3957                 "spanNulls": true,
3958                 "stacking": {
3959                   "group": "A",
3960                   "mode": "none"
3961                 },
3962                 "thresholdsStyle": {
3963                   "mode": "off"
3964                 }
3965               },
3966               "links": [],
3967               "mappings": [],
3968               "thresholds": {
3969                 "mode": "absolute",
3970                 "steps": [
3971                   {
3972                     "color": "green"
3973                   },
3974                   {
3975                     "color": "red",
3976                     "value": 80
3977                   }
3978                 ]
3979               },
3980               "unit": "ms"
3981             },
3982             "overrides": []
3983           },
3984           "gridPos": {
3985             "h": 10,
3986             "w": 8,
3987             "x": 8,
3988             "y": 38
3989           },
3990           "id": 63,
3991           "options": {
3992             "legend": {
3993               "calcs": [
3994                 "mean",
3995                 "lastNotNull",
3996                 "max"
3997               ],
3998               "displayMode": "table",
3999               "placement": "bottom",
4000               "showLegend": true
4001             },
4002             "tooltip": {
4003               "mode": "single"
4004             }
4005           },
4006           "pluginVersion": "8.1.3",
4007           "targets": [
4008             {
4009               "datasource": {
4010                 "type": "prometheus",
4011                 "uid": "PBFA97CFB590B2093"
4012               },
4013               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
4014               "hide": false,
4015               "legendFormat": "{{instance}} - {{quantile}}",
4016               "refId": "A"
4017             }
4018           ],
4019           "title": "Producer - ResponseQueueTimeMs",
4020           "type": "timeseries"
4021         },
4022         {
4023           "datasource": {
4024             "type": "prometheus",
4025             "uid": "PBFA97CFB590B2093"
4026           },
4027           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4028           "fieldConfig": {
4029             "defaults": {
4030               "color": {
4031                 "mode": "palette-classic"
4032               },
4033               "custom": {
4034                 "axisLabel": "",
4035                 "axisPlacement": "auto",
4036                 "barAlignment": 0,
4037                 "drawStyle": "line",
4038                 "fillOpacity": 10,
4039                 "gradientMode": "none",
4040                 "hideFrom": {
4041                   "legend": false,
4042                   "tooltip": false,
4043                   "viz": false
4044                 },
4045                 "lineInterpolation": "linear",
4046                 "lineWidth": 1,
4047                 "pointSize": 5,
4048                 "scaleDistribution": {
4049                   "type": "linear"
4050                 },
4051                 "showPoints": "never",
4052                 "spanNulls": true,
4053                 "stacking": {
4054                   "group": "A",
4055                   "mode": "none"
4056                 },
4057                 "thresholdsStyle": {
4058                   "mode": "off"
4059                 }
4060               },
4061               "links": [],
4062               "mappings": [],
4063               "min": 0,
4064               "thresholds": {
4065                 "mode": "absolute",
4066                 "steps": [
4067                   {
4068                     "color": "green"
4069                   },
4070                   {
4071                     "color": "red",
4072                     "value": 80
4073                   }
4074                 ]
4075               },
4076               "unit": "ms"
4077             },
4078             "overrides": []
4079           },
4080           "gridPos": {
4081             "h": 10,
4082             "w": 8,
4083             "x": 16,
4084             "y": 38
4085           },
4086           "id": 64,
4087           "options": {
4088             "legend": {
4089               "calcs": [
4090                 "mean",
4091                 "lastNotNull",
4092                 "max"
4093               ],
4094               "displayMode": "table",
4095               "placement": "bottom",
4096               "showLegend": true
4097             },
4098             "tooltip": {
4099               "mode": "single"
4100             }
4101           },
4102           "pluginVersion": "8.1.3",
4103           "targets": [
4104             {
4105               "datasource": {
4106                 "type": "prometheus",
4107                 "uid": "PBFA97CFB590B2093"
4108               },
4109               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Produce\"}",
4110               "hide": false,
4111               "legendFormat": "{{instance}} - {{quantile}}",
4112               "refId": "A"
4113             }
4114           ],
4115           "title": "Producer - ResponseSendTimeMs",
4116           "type": "timeseries"
4117         }
4118       ],
4119       "targets": [
4120         {
4121           "datasource": {
4122             "type": "datasource",
4123             "uid": "grafana"
4124           },
4125           "refId": "A"
4126         }
4127       ],
4128       "title": "Producer Performance",
4129       "type": "row"
4130     },
4131     {
4132       "collapsed": true,
4133       "datasource": {
4134         "type": "datasource",
4135         "uid": "grafana"
4136       },
4137       "gridPos": {
4138         "h": 1,
4139         "w": 24,
4140         "x": 0,
4141         "y": 51
4142       },
4143       "id": 68,
4144       "panels": [
4145         {
4146           "datasource": {
4147             "type": "prometheus",
4148             "uid": "PBFA97CFB590B2093"
4149           },
4150           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4151           "fieldConfig": {
4152             "defaults": {
4153               "color": {
4154                 "mode": "palette-classic"
4155               },
4156               "custom": {
4157                 "axisLabel": "",
4158                 "axisPlacement": "auto",
4159                 "barAlignment": 0,
4160                 "drawStyle": "line",
4161                 "fillOpacity": 10,
4162                 "gradientMode": "none",
4163                 "hideFrom": {
4164                   "legend": false,
4165                   "tooltip": false,
4166                   "viz": false
4167                 },
4168                 "lineInterpolation": "linear",
4169                 "lineWidth": 1,
4170                 "pointSize": 5,
4171                 "scaleDistribution": {
4172                   "type": "linear"
4173                 },
4174                 "showPoints": "never",
4175                 "spanNulls": true,
4176                 "stacking": {
4177                   "group": "A",
4178                   "mode": "none"
4179                 },
4180                 "thresholdsStyle": {
4181                   "mode": "off"
4182                 }
4183               },
4184               "links": [],
4185               "mappings": [],
4186               "min": 0,
4187               "thresholds": {
4188                 "mode": "absolute",
4189                 "steps": [
4190                   {
4191                     "color": "green"
4192                   },
4193                   {
4194                     "color": "red",
4195                     "value": 80
4196                   }
4197                 ]
4198               },
4199               "unit": "ms"
4200             },
4201             "overrides": []
4202           },
4203           "gridPos": {
4204             "h": 9,
4205             "w": 12,
4206             "x": 0,
4207             "y": 30
4208           },
4209           "id": 69,
4210           "options": {
4211             "legend": {
4212               "calcs": [
4213                 "mean",
4214                 "lastNotNull",
4215                 "max"
4216               ],
4217               "displayMode": "list",
4218               "placement": "bottom",
4219               "showLegend": true
4220             },
4221             "tooltip": {
4222               "mode": "single"
4223             }
4224           },
4225           "pluginVersion": "8.1.3",
4226           "targets": [
4227             {
4228               "datasource": {
4229                 "type": "prometheus",
4230                 "uid": "PBFA97CFB590B2093"
4231               },
4232               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4233               "hide": false,
4234               "legendFormat": "{{instance}} - {{quantile}}",
4235               "refId": "A"
4236             }
4237           ],
4238           "title": "Consumer - RequestQueueTimeMs",
4239           "type": "timeseries"
4240         },
4241         {
4242           "datasource": {
4243             "type": "prometheus",
4244             "uid": "PBFA97CFB590B2093"
4245           },
4246           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4247           "fieldConfig": {
4248             "defaults": {
4249               "color": {
4250                 "mode": "palette-classic"
4251               },
4252               "custom": {
4253                 "axisLabel": "",
4254                 "axisPlacement": "auto",
4255                 "barAlignment": 0,
4256                 "drawStyle": "line",
4257                 "fillOpacity": 10,
4258                 "gradientMode": "none",
4259                 "hideFrom": {
4260                   "legend": false,
4261                   "tooltip": false,
4262                   "viz": false
4263                 },
4264                 "lineInterpolation": "linear",
4265                 "lineWidth": 1,
4266                 "pointSize": 5,
4267                 "scaleDistribution": {
4268                   "type": "linear"
4269                 },
4270                 "showPoints": "never",
4271                 "spanNulls": true,
4272                 "stacking": {
4273                   "group": "A",
4274                   "mode": "none"
4275                 },
4276                 "thresholdsStyle": {
4277                   "mode": "off"
4278                 }
4279               },
4280               "links": [],
4281               "mappings": [],
4282               "thresholds": {
4283                 "mode": "absolute",
4284                 "steps": [
4285                   {
4286                     "color": "green"
4287                   },
4288                   {
4289                     "color": "red",
4290                     "value": 80
4291                   }
4292                 ]
4293               },
4294               "unit": "ms"
4295             },
4296             "overrides": []
4297           },
4298           "gridPos": {
4299             "h": 9,
4300             "w": 12,
4301             "x": 12,
4302             "y": 30
4303           },
4304           "id": 70,
4305           "options": {
4306             "legend": {
4307               "calcs": [
4308                 "mean",
4309                 "lastNotNull",
4310                 "max"
4311               ],
4312               "displayMode": "list",
4313               "placement": "bottom",
4314               "showLegend": true
4315             },
4316             "tooltip": {
4317               "mode": "single"
4318             }
4319           },
4320           "pluginVersion": "8.1.3",
4321           "targets": [
4322             {
4323               "datasource": {
4324                 "type": "prometheus",
4325                 "uid": "PBFA97CFB590B2093"
4326               },
4327               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4328               "hide": false,
4329               "legendFormat": "{{instance}} - {{quantile}}",
4330               "refId": "A"
4331             }
4332           ],
4333           "title": "Consumer - LocalTimeMs",
4334           "type": "timeseries"
4335         },
4336         {
4337           "datasource": {
4338             "type": "prometheus",
4339             "uid": "PBFA97CFB590B2093"
4340           },
4341           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4342           "fieldConfig": {
4343             "defaults": {
4344               "color": {
4345                 "mode": "palette-classic"
4346               },
4347               "custom": {
4348                 "axisLabel": "",
4349                 "axisPlacement": "auto",
4350                 "barAlignment": 0,
4351                 "drawStyle": "line",
4352                 "fillOpacity": 10,
4353                 "gradientMode": "none",
4354                 "hideFrom": {
4355                   "legend": false,
4356                   "tooltip": false,
4357                   "viz": false
4358                 },
4359                 "lineInterpolation": "linear",
4360                 "lineWidth": 1,
4361                 "pointSize": 5,
4362                 "scaleDistribution": {
4363                   "type": "linear"
4364                 },
4365                 "showPoints": "never",
4366                 "spanNulls": true,
4367                 "stacking": {
4368                   "group": "A",
4369                   "mode": "none"
4370                 },
4371                 "thresholdsStyle": {
4372                   "mode": "off"
4373                 }
4374               },
4375               "links": [],
4376               "mappings": [],
4377               "thresholds": {
4378                 "mode": "absolute",
4379                 "steps": [
4380                   {
4381                     "color": "green"
4382                   },
4383                   {
4384                     "color": "red",
4385                     "value": 80
4386                   }
4387                 ]
4388               },
4389               "unit": "ms"
4390             },
4391             "overrides": []
4392           },
4393           "gridPos": {
4394             "h": 9,
4395             "w": 8,
4396             "x": 0,
4397             "y": 39
4398           },
4399           "id": 71,
4400           "options": {
4401             "legend": {
4402               "calcs": [
4403                 "mean",
4404                 "lastNotNull",
4405                 "max"
4406               ],
4407               "displayMode": "table",
4408               "placement": "bottom",
4409               "showLegend": true
4410             },
4411             "tooltip": {
4412               "mode": "single"
4413             }
4414           },
4415           "pluginVersion": "8.1.3",
4416           "targets": [
4417             {
4418               "datasource": {
4419                 "type": "prometheus",
4420                 "uid": "PBFA97CFB590B2093"
4421               },
4422               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4423               "hide": false,
4424               "legendFormat": "{{instance}} - {{quantile}}",
4425               "refId": "A"
4426             }
4427           ],
4428           "title": "Consumer - RemoteTimeMs",
4429           "type": "timeseries"
4430         },
4431         {
4432           "datasource": {
4433             "type": "prometheus",
4434             "uid": "PBFA97CFB590B2093"
4435           },
4436           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4437           "fieldConfig": {
4438             "defaults": {
4439               "color": {
4440                 "mode": "palette-classic"
4441               },
4442               "custom": {
4443                 "axisLabel": "",
4444                 "axisPlacement": "auto",
4445                 "barAlignment": 0,
4446                 "drawStyle": "line",
4447                 "fillOpacity": 10,
4448                 "gradientMode": "none",
4449                 "hideFrom": {
4450                   "legend": false,
4451                   "tooltip": false,
4452                   "viz": false
4453                 },
4454                 "lineInterpolation": "linear",
4455                 "lineWidth": 1,
4456                 "pointSize": 5,
4457                 "scaleDistribution": {
4458                   "type": "linear"
4459                 },
4460                 "showPoints": "never",
4461                 "spanNulls": true,
4462                 "stacking": {
4463                   "group": "A",
4464                   "mode": "none"
4465                 },
4466                 "thresholdsStyle": {
4467                   "mode": "off"
4468                 }
4469               },
4470               "links": [],
4471               "mappings": [],
4472               "min": 0,
4473               "thresholds": {
4474                 "mode": "absolute",
4475                 "steps": [
4476                   {
4477                     "color": "green"
4478                   },
4479                   {
4480                     "color": "red",
4481                     "value": 80
4482                   }
4483                 ]
4484               },
4485               "unit": "ms"
4486             },
4487             "overrides": []
4488           },
4489           "gridPos": {
4490             "h": 9,
4491             "w": 8,
4492             "x": 8,
4493             "y": 39
4494           },
4495           "id": 72,
4496           "options": {
4497             "legend": {
4498               "calcs": [
4499                 "mean",
4500                 "lastNotNull",
4501                 "max"
4502               ],
4503               "displayMode": "table",
4504               "placement": "bottom",
4505               "showLegend": true
4506             },
4507             "tooltip": {
4508               "mode": "single"
4509             }
4510           },
4511           "pluginVersion": "8.1.3",
4512           "targets": [
4513             {
4514               "datasource": {
4515                 "type": "prometheus",
4516                 "uid": "PBFA97CFB590B2093"
4517               },
4518               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4519               "hide": false,
4520               "legendFormat": "{{instance}} - {{quantile}}",
4521               "refId": "A"
4522             }
4523           ],
4524           "title": "Consumer - ResponseQueueTimeMs",
4525           "type": "timeseries"
4526         },
4527         {
4528           "datasource": {
4529             "type": "prometheus",
4530             "uid": "PBFA97CFB590B2093"
4531           },
4532           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
4533           "fieldConfig": {
4534             "defaults": {
4535               "color": {
4536                 "mode": "palette-classic"
4537               },
4538               "custom": {
4539                 "axisLabel": "",
4540                 "axisPlacement": "auto",
4541                 "barAlignment": 0,
4542                 "drawStyle": "line",
4543                 "fillOpacity": 10,
4544                 "gradientMode": "none",
4545                 "hideFrom": {
4546                   "legend": false,
4547                   "tooltip": false,
4548                   "viz": false
4549                 },
4550                 "lineInterpolation": "linear",
4551                 "lineWidth": 1,
4552                 "pointSize": 5,
4553                 "scaleDistribution": {
4554                   "type": "linear"
4555                 },
4556                 "showPoints": "never",
4557                 "spanNulls": true,
4558                 "stacking": {
4559                   "group": "A",
4560                   "mode": "none"
4561                 },
4562                 "thresholdsStyle": {
4563                   "mode": "off"
4564                 }
4565               },
4566               "links": [],
4567               "mappings": [],
4568               "min": 0,
4569               "thresholds": {
4570                 "mode": "absolute",
4571                 "steps": [
4572                   {
4573                     "color": "green"
4574                   },
4575                   {
4576                     "color": "red",
4577                     "value": 80
4578                   }
4579                 ]
4580               },
4581               "unit": "ms"
4582             },
4583             "overrides": []
4584           },
4585           "gridPos": {
4586             "h": 9,
4587             "w": 8,
4588             "x": 16,
4589             "y": 39
4590           },
4591           "id": 73,
4592           "options": {
4593             "legend": {
4594               "calcs": [
4595                 "mean",
4596                 "lastNotNull",
4597                 "max"
4598               ],
4599               "displayMode": "table",
4600               "placement": "bottom",
4601               "showLegend": true
4602             },
4603             "tooltip": {
4604               "mode": "single"
4605             }
4606           },
4607           "pluginVersion": "8.1.3",
4608           "targets": [
4609             {
4610               "datasource": {
4611                 "type": "prometheus",
4612                 "uid": "PBFA97CFB590B2093"
4613               },
4614               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"Fetch\"}",
4615               "hide": false,
4616               "legendFormat": "{{instance}} - {{quantile}}",
4617               "refId": "A"
4618             }
4619           ],
4620           "title": "Consumer - ResponseSendTimeMs",
4621           "type": "timeseries"
4622         }
4623       ],
4624       "targets": [
4625         {
4626           "datasource": {
4627             "type": "datasource",
4628             "uid": "grafana"
4629           },
4630           "refId": "A"
4631         }
4632       ],
4633       "title": "Consumer Performance",
4634       "type": "row"
4635     },
4636     {
4637       "collapsed": true,
4638       "datasource": {
4639         "type": "datasource",
4640         "uid": "grafana"
4641       },
4642       "gridPos": {
4643         "h": 1,
4644         "w": 24,
4645         "x": 0,
4646         "y": 52
4647       },
4648       "id": 66,
4649       "panels": [
4650         {
4651           "datasource": {
4652             "type": "prometheus",
4653             "uid": "PBFA97CFB590B2093"
4654           },
4655           "description": "A high value can imply there aren't enough IO threads or the CPU is a bottleneck, or the request queue isnt large enough. The request queue size should match the number of connections.",
4656           "fieldConfig": {
4657             "defaults": {
4658               "color": {
4659                 "mode": "palette-classic"
4660               },
4661               "custom": {
4662                 "axisLabel": "",
4663                 "axisPlacement": "auto",
4664                 "barAlignment": 0,
4665                 "drawStyle": "line",
4666                 "fillOpacity": 10,
4667                 "gradientMode": "none",
4668                 "hideFrom": {
4669                   "legend": false,
4670                   "tooltip": false,
4671                   "viz": false
4672                 },
4673                 "lineInterpolation": "linear",
4674                 "lineWidth": 1,
4675                 "pointSize": 5,
4676                 "scaleDistribution": {
4677                   "type": "linear"
4678                 },
4679                 "showPoints": "never",
4680                 "spanNulls": true,
4681                 "stacking": {
4682                   "group": "A",
4683                   "mode": "none"
4684                 },
4685                 "thresholdsStyle": {
4686                   "mode": "off"
4687                 }
4688               },
4689               "links": [],
4690               "mappings": [],
4691               "min": 0,
4692               "thresholds": {
4693                 "mode": "absolute",
4694                 "steps": [
4695                   {
4696                     "color": "green"
4697                   },
4698                   {
4699                     "color": "red",
4700                     "value": 80
4701                   }
4702                 ]
4703               },
4704               "unit": "ms"
4705             },
4706             "overrides": []
4707           },
4708           "gridPos": {
4709             "h": 9,
4710             "w": 12,
4711             "x": 0,
4712             "y": 31
4713           },
4714           "id": 74,
4715           "options": {
4716             "legend": {
4717               "calcs": [
4718                 "mean",
4719                 "lastNotNull",
4720                 "max"
4721               ],
4722               "displayMode": "list",
4723               "placement": "bottom",
4724               "showLegend": true
4725             },
4726             "tooltip": {
4727               "mode": "single"
4728             }
4729           },
4730           "pluginVersion": "8.1.3",
4731           "targets": [
4732             {
4733               "datasource": {
4734                 "type": "prometheus",
4735                 "uid": "PBFA97CFB590B2093"
4736               },
4737               "expr": "kafka_network_requestmetrics_requestqueuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4738               "hide": false,
4739               "legendFormat": "{{instance}} - {{quantile}}",
4740               "refId": "A"
4741             }
4742           ],
4743           "title": "FetchFollower - RequestQueueTimeMs",
4744           "type": "timeseries"
4745         },
4746         {
4747           "datasource": {
4748             "type": "prometheus",
4749             "uid": "PBFA97CFB590B2093"
4750           },
4751           "description": "In most cases, a high value can imply slow local storage or the storage is a bottleneck. One should also investigate LogFlushRateAndTimeMs to know how long page flushes are taking, which will also indicate a slow disk. In the case of FetchFollower requests, time spent in LocalTimeMs can be the result of a ZooKeeper write to change the ISR.",
4752           "fieldConfig": {
4753             "defaults": {
4754               "color": {
4755                 "mode": "palette-classic"
4756               },
4757               "custom": {
4758                 "axisLabel": "",
4759                 "axisPlacement": "auto",
4760                 "barAlignment": 0,
4761                 "drawStyle": "line",
4762                 "fillOpacity": 10,
4763                 "gradientMode": "none",
4764                 "hideFrom": {
4765                   "legend": false,
4766                   "tooltip": false,
4767                   "viz": false
4768                 },
4769                 "lineInterpolation": "linear",
4770                 "lineWidth": 1,
4771                 "pointSize": 5,
4772                 "scaleDistribution": {
4773                   "type": "linear"
4774                 },
4775                 "showPoints": "never",
4776                 "spanNulls": true,
4777                 "stacking": {
4778                   "group": "A",
4779                   "mode": "none"
4780                 },
4781                 "thresholdsStyle": {
4782                   "mode": "off"
4783                 }
4784               },
4785               "links": [],
4786               "mappings": [],
4787               "min": 0,
4788               "thresholds": {
4789                 "mode": "absolute",
4790                 "steps": [
4791                   {
4792                     "color": "green"
4793                   },
4794                   {
4795                     "color": "red",
4796                     "value": 80
4797                   }
4798                 ]
4799               },
4800               "unit": "ms"
4801             },
4802             "overrides": []
4803           },
4804           "gridPos": {
4805             "h": 9,
4806             "w": 12,
4807             "x": 12,
4808             "y": 31
4809           },
4810           "id": 75,
4811           "options": {
4812             "legend": {
4813               "calcs": [
4814                 "mean",
4815                 "lastNotNull",
4816                 "max"
4817               ],
4818               "displayMode": "list",
4819               "placement": "bottom",
4820               "showLegend": true
4821             },
4822             "tooltip": {
4823               "mode": "single"
4824             }
4825           },
4826           "pluginVersion": "8.1.3",
4827           "targets": [
4828             {
4829               "datasource": {
4830                 "type": "prometheus",
4831                 "uid": "PBFA97CFB590B2093"
4832               },
4833               "expr": "kafka_network_requestmetrics_localtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4834               "hide": false,
4835               "legendFormat": "{{instance}} - {{quantile}}",
4836               "refId": "A"
4837             }
4838           ],
4839           "title": "FetchFollower - LocalTimeMs",
4840           "type": "timeseries"
4841         },
4842         {
4843           "datasource": {
4844             "type": "prometheus",
4845             "uid": "PBFA97CFB590B2093"
4846           },
4847           "description": "A high value can imply a slow network connection. For fetch request, if the remote time is high, it could be that there is not enough data to give in a fetch response. This can happen when the consumer or replica is caught up and there is no new incoming data. If this is the case, remote time will be close to the max wait time, which is normal. Max wait time is configured via replica.fetch.wait.max.ms and fetch.max.wait.ms.\n",
4848           "fieldConfig": {
4849             "defaults": {
4850               "color": {
4851                 "mode": "palette-classic"
4852               },
4853               "custom": {
4854                 "axisLabel": "",
4855                 "axisPlacement": "auto",
4856                 "barAlignment": 0,
4857                 "drawStyle": "line",
4858                 "fillOpacity": 10,
4859                 "gradientMode": "none",
4860                 "hideFrom": {
4861                   "legend": false,
4862                   "tooltip": false,
4863                   "viz": false
4864                 },
4865                 "lineInterpolation": "linear",
4866                 "lineWidth": 1,
4867                 "pointSize": 5,
4868                 "scaleDistribution": {
4869                   "type": "linear"
4870                 },
4871                 "showPoints": "never",
4872                 "spanNulls": true,
4873                 "stacking": {
4874                   "group": "A",
4875                   "mode": "none"
4876                 },
4877                 "thresholdsStyle": {
4878                   "mode": "off"
4879                 }
4880               },
4881               "links": [],
4882               "mappings": [],
4883               "thresholds": {
4884                 "mode": "absolute",
4885                 "steps": [
4886                   {
4887                     "color": "green"
4888                   },
4889                   {
4890                     "color": "red",
4891                     "value": 80
4892                   }
4893                 ]
4894               },
4895               "unit": "ms"
4896             },
4897             "overrides": []
4898           },
4899           "gridPos": {
4900             "h": 9,
4901             "w": 8,
4902             "x": 0,
4903             "y": 40
4904           },
4905           "id": 76,
4906           "options": {
4907             "legend": {
4908               "calcs": [
4909                 "mean",
4910                 "lastNotNull",
4911                 "max"
4912               ],
4913               "displayMode": "table",
4914               "placement": "bottom",
4915               "showLegend": true
4916             },
4917             "tooltip": {
4918               "mode": "single"
4919             }
4920           },
4921           "pluginVersion": "8.1.3",
4922           "targets": [
4923             {
4924               "datasource": {
4925                 "type": "prometheus",
4926                 "uid": "PBFA97CFB590B2093"
4927               },
4928               "expr": "kafka_network_requestmetrics_remotetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
4929               "hide": false,
4930               "legendFormat": "{{instance}} - {{quantile}}",
4931               "refId": "A"
4932             }
4933           ],
4934           "title": "FetchFollower - RemoteTimeMs",
4935           "type": "timeseries"
4936         },
4937         {
4938           "datasource": {
4939             "type": "prometheus",
4940             "uid": "PBFA97CFB590B2093"
4941           },
4942           "description": "A high value can imply there aren't enough network threads or the network cant dequeue responses quickly enough, causing back pressure in the response queue.\n",
4943           "fieldConfig": {
4944             "defaults": {
4945               "color": {
4946                 "mode": "palette-classic"
4947               },
4948               "custom": {
4949                 "axisLabel": "",
4950                 "axisPlacement": "auto",
4951                 "barAlignment": 0,
4952                 "drawStyle": "line",
4953                 "fillOpacity": 10,
4954                 "gradientMode": "none",
4955                 "hideFrom": {
4956                   "legend": false,
4957                   "tooltip": false,
4958                   "viz": false
4959                 },
4960                 "lineInterpolation": "linear",
4961                 "lineWidth": 1,
4962                 "pointSize": 5,
4963                 "scaleDistribution": {
4964                   "type": "linear"
4965                 },
4966                 "showPoints": "never",
4967                 "spanNulls": true,
4968                 "stacking": {
4969                   "group": "A",
4970                   "mode": "none"
4971                 },
4972                 "thresholdsStyle": {
4973                   "mode": "off"
4974                 }
4975               },
4976               "links": [],
4977               "mappings": [],
4978               "thresholds": {
4979                 "mode": "absolute",
4980                 "steps": [
4981                   {
4982                     "color": "green"
4983                   },
4984                   {
4985                     "color": "red",
4986                     "value": 80
4987                   }
4988                 ]
4989               },
4990               "unit": "ms"
4991             },
4992             "overrides": []
4993           },
4994           "gridPos": {
4995             "h": 9,
4996             "w": 8,
4997             "x": 8,
4998             "y": 40
4999           },
5000           "id": 77,
5001           "options": {
5002             "legend": {
5003               "calcs": [
5004                 "mean",
5005                 "lastNotNull",
5006                 "max"
5007               ],
5008               "displayMode": "table",
5009               "placement": "bottom",
5010               "showLegend": true
5011             },
5012             "tooltip": {
5013               "mode": "single"
5014             }
5015           },
5016           "pluginVersion": "8.1.3",
5017           "targets": [
5018             {
5019               "datasource": {
5020                 "type": "prometheus",
5021                 "uid": "PBFA97CFB590B2093"
5022               },
5023               "expr": "kafka_network_requestmetrics_responsequeuetimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
5024               "hide": false,
5025               "legendFormat": "{{instance}} - {{quantile}}",
5026               "refId": "A"
5027             }
5028           ],
5029           "title": "FetchFollower - ResponseQueueTimeMs",
5030           "type": "timeseries"
5031         },
5032         {
5033           "datasource": {
5034             "type": "prometheus",
5035             "uid": "PBFA97CFB590B2093"
5036           },
5037           "description": "A high value can imply the zero-copy from disk to the network is slow, or the network is the bottleneck because the network cant dequeue responses of the TCP socket as quickly as theyre being created. If the network buffer gets full, Kafka will block.\n",
5038           "fieldConfig": {
5039             "defaults": {
5040               "color": {
5041                 "mode": "palette-classic"
5042               },
5043               "custom": {
5044                 "axisLabel": "",
5045                 "axisPlacement": "auto",
5046                 "barAlignment": 0,
5047                 "drawStyle": "line",
5048                 "fillOpacity": 10,
5049                 "gradientMode": "none",
5050                 "hideFrom": {
5051                   "legend": false,
5052                   "tooltip": false,
5053                   "viz": false
5054                 },
5055                 "lineInterpolation": "linear",
5056                 "lineWidth": 1,
5057                 "pointSize": 5,
5058                 "scaleDistribution": {
5059                   "type": "linear"
5060                 },
5061                 "showPoints": "never",
5062                 "spanNulls": true,
5063                 "stacking": {
5064                   "group": "A",
5065                   "mode": "none"
5066                 },
5067                 "thresholdsStyle": {
5068                   "mode": "off"
5069                 }
5070               },
5071               "links": [],
5072               "mappings": [],
5073               "min": 0,
5074               "thresholds": {
5075                 "mode": "absolute",
5076                 "steps": [
5077                   {
5078                     "color": "green"
5079                   },
5080                   {
5081                     "color": "red",
5082                     "value": 80
5083                   }
5084                 ]
5085               },
5086               "unit": "ms"
5087             },
5088             "overrides": []
5089           },
5090           "gridPos": {
5091             "h": 9,
5092             "w": 8,
5093             "x": 16,
5094             "y": 40
5095           },
5096           "id": 78,
5097           "options": {
5098             "legend": {
5099               "calcs": [
5100                 "mean",
5101                 "lastNotNull",
5102                 "max"
5103               ],
5104               "displayMode": "table",
5105               "placement": "bottom",
5106               "showLegend": true
5107             },
5108             "tooltip": {
5109               "mode": "single"
5110             }
5111           },
5112           "pluginVersion": "8.1.3",
5113           "targets": [
5114             {
5115               "datasource": {
5116                 "type": "prometheus",
5117                 "uid": "PBFA97CFB590B2093"
5118               },
5119               "expr": "kafka_network_requestmetrics_responsesendtimems{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\",quantile=~\"$percentile\",request=\"FetchFollower\"}",
5120               "hide": false,
5121               "legendFormat": "{{instance}} - {{quantile}}",
5122               "refId": "A"
5123             }
5124           ],
5125           "title": "FetchFollower - ResponseSendTimeMs",
5126           "type": "timeseries"
5127         }
5128       ],
5129       "targets": [
5130         {
5131           "datasource": {
5132             "type": "datasource",
5133             "uid": "grafana"
5134           },
5135           "refId": "A"
5136         }
5137       ],
5138       "title": "Fetch Follower Performance",
5139       "type": "row"
5140     },
5141     {
5142       "collapsed": true,
5143       "datasource": {
5144         "type": "datasource",
5145         "uid": "grafana"
5146       },
5147       "gridPos": {
5148         "h": 1,
5149         "w": 24,
5150         "x": 0,
5151         "y": 53
5152       },
5153       "id": 102,
5154       "panels": [
5155         {
5156           "datasource": {
5157             "type": "prometheus",
5158             "uid": "PBFA97CFB590B2093"
5159           },
5160           "fieldConfig": {
5161             "defaults": {
5162               "color": {
5163                 "mode": "palette-classic"
5164               },
5165               "custom": {
5166                 "axisLabel": "",
5167                 "axisPlacement": "auto",
5168                 "barAlignment": 0,
5169                 "drawStyle": "line",
5170                 "fillOpacity": 10,
5171                 "gradientMode": "none",
5172                 "hideFrom": {
5173                   "legend": false,
5174                   "tooltip": false,
5175                   "viz": false
5176                 },
5177                 "lineInterpolation": "linear",
5178                 "lineWidth": 1,
5179                 "pointSize": 5,
5180                 "scaleDistribution": {
5181                   "type": "linear"
5182                 },
5183                 "showPoints": "never",
5184                 "spanNulls": true,
5185                 "stacking": {
5186                   "group": "A",
5187                   "mode": "none"
5188                 },
5189                 "thresholdsStyle": {
5190                   "mode": "off"
5191                 }
5192               },
5193               "links": [],
5194               "mappings": [],
5195               "thresholds": {
5196                 "mode": "absolute",
5197                 "steps": [
5198                   {
5199                     "color": "green"
5200                   },
5201                   {
5202                     "color": "red",
5203                     "value": 80
5204                   }
5205                 ]
5206               },
5207               "unit": "short"
5208             },
5209             "overrides": []
5210           },
5211           "gridPos": {
5212             "h": 8,
5213             "w": 12,
5214             "x": 0,
5215             "y": 32
5216           },
5217           "id": 98,
5218           "options": {
5219             "legend": {
5220               "calcs": [
5221                 "mean",
5222                 "lastNotNull",
5223                 "max"
5224               ],
5225               "displayMode": "table",
5226               "placement": "bottom",
5227               "showLegend": true
5228             },
5229             "tooltip": {
5230               "mode": "single"
5231             }
5232           },
5233           "pluginVersion": "8.1.3",
5234           "targets": [
5235             {
5236               "datasource": {
5237                 "type": "prometheus",
5238                 "uid": "PBFA97CFB590B2093"
5239               },
5240               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5241               "interval": "",
5242               "legendFormat": "{{listener}}",
5243               "refId": "A"
5244             }
5245           ],
5246           "title": "Connections count per listener",
5247           "type": "timeseries"
5248         },
5249         {
5250           "datasource": {
5251             "type": "prometheus",
5252             "uid": "PBFA97CFB590B2093"
5253           },
5254           "fieldConfig": {
5255             "defaults": {
5256               "color": {
5257                 "mode": "palette-classic"
5258               },
5259               "custom": {
5260                 "axisLabel": "",
5261                 "axisPlacement": "auto",
5262                 "barAlignment": 0,
5263                 "drawStyle": "line",
5264                 "fillOpacity": 10,
5265                 "gradientMode": "none",
5266                 "hideFrom": {
5267                   "legend": false,
5268                   "tooltip": false,
5269                   "viz": false
5270                 },
5271                 "lineInterpolation": "linear",
5272                 "lineWidth": 1,
5273                 "pointSize": 5,
5274                 "scaleDistribution": {
5275                   "type": "linear"
5276                 },
5277                 "showPoints": "never",
5278                 "spanNulls": true,
5279                 "stacking": {
5280                   "group": "A",
5281                   "mode": "none"
5282                 },
5283                 "thresholdsStyle": {
5284                   "mode": "off"
5285                 }
5286               },
5287               "links": [],
5288               "mappings": [],
5289               "thresholds": {
5290                 "mode": "absolute",
5291                 "steps": [
5292                   {
5293                     "color": "green"
5294                   },
5295                   {
5296                     "color": "red",
5297                     "value": 80
5298                   }
5299                 ]
5300               },
5301               "unit": "short"
5302             },
5303             "overrides": []
5304           },
5305           "gridPos": {
5306             "h": 8,
5307             "w": 12,
5308             "x": 12,
5309             "y": 32
5310           },
5311           "id": 100,
5312           "options": {
5313             "legend": {
5314               "calcs": [
5315                 "mean",
5316                 "lastNotNull",
5317                 "max"
5318               ],
5319               "displayMode": "table",
5320               "placement": "bottom",
5321               "showLegend": true
5322             },
5323             "tooltip": {
5324               "mode": "single"
5325             }
5326           },
5327           "pluginVersion": "8.1.3",
5328           "targets": [
5329             {
5330               "datasource": {
5331                 "type": "prometheus",
5332                 "uid": "PBFA97CFB590B2093"
5333               },
5334               "expr": "sum(kafka_server_socketservermetrics_connection_count{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5335               "interval": "",
5336               "legendFormat": "{{instance}}",
5337               "refId": "A"
5338             }
5339           ],
5340           "title": "Connections count per broker",
5341           "type": "timeseries"
5342         },
5343         {
5344           "datasource": {
5345             "type": "prometheus",
5346             "uid": "PBFA97CFB590B2093"
5347           },
5348           "fieldConfig": {
5349             "defaults": {
5350               "color": {
5351                 "mode": "palette-classic"
5352               },
5353               "custom": {
5354                 "axisLabel": "",
5355                 "axisPlacement": "auto",
5356                 "barAlignment": 0,
5357                 "drawStyle": "line",
5358                 "fillOpacity": 10,
5359                 "gradientMode": "none",
5360                 "hideFrom": {
5361                   "legend": false,
5362                   "tooltip": false,
5363                   "viz": false
5364                 },
5365                 "lineInterpolation": "linear",
5366                 "lineWidth": 1,
5367                 "pointSize": 5,
5368                 "scaleDistribution": {
5369                   "type": "linear"
5370                 },
5371                 "showPoints": "never",
5372                 "spanNulls": true,
5373                 "stacking": {
5374                   "group": "A",
5375                   "mode": "none"
5376                 },
5377                 "thresholdsStyle": {
5378                   "mode": "off"
5379                 }
5380               },
5381               "links": [],
5382               "mappings": [],
5383               "thresholds": {
5384                 "mode": "absolute",
5385                 "steps": [
5386                   {
5387                     "color": "green"
5388                   },
5389                   {
5390                     "color": "red",
5391                     "value": 80
5392                   }
5393                 ]
5394               },
5395               "unit": "short"
5396             },
5397             "overrides": []
5398           },
5399           "gridPos": {
5400             "h": 8,
5401             "w": 12,
5402             "x": 0,
5403             "y": 40
5404           },
5405           "id": 104,
5406           "options": {
5407             "legend": {
5408               "calcs": [
5409                 "mean",
5410                 "lastNotNull",
5411                 "max"
5412               ],
5413               "displayMode": "table",
5414               "placement": "bottom",
5415               "showLegend": true
5416             },
5417             "tooltip": {
5418               "mode": "single"
5419             }
5420           },
5421           "pluginVersion": "8.1.3",
5422           "targets": [
5423             {
5424               "datasource": {
5425                 "type": "prometheus",
5426                 "uid": "PBFA97CFB590B2093"
5427               },
5428               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5429               "interval": "",
5430               "legendFormat": "{{listener}}",
5431               "refId": "A"
5432             }
5433           ],
5434           "title": "Connections creation rate per listener",
5435           "type": "timeseries"
5436         },
5437         {
5438           "datasource": {
5439             "type": "prometheus",
5440             "uid": "PBFA97CFB590B2093"
5441           },
5442           "fieldConfig": {
5443             "defaults": {
5444               "color": {
5445                 "mode": "palette-classic"
5446               },
5447               "custom": {
5448                 "axisLabel": "",
5449                 "axisPlacement": "auto",
5450                 "barAlignment": 0,
5451                 "drawStyle": "line",
5452                 "fillOpacity": 10,
5453                 "gradientMode": "none",
5454                 "hideFrom": {
5455                   "legend": false,
5456                   "tooltip": false,
5457                   "viz": false
5458                 },
5459                 "lineInterpolation": "linear",
5460                 "lineWidth": 1,
5461                 "pointSize": 5,
5462                 "scaleDistribution": {
5463                   "type": "linear"
5464                 },
5465                 "showPoints": "never",
5466                 "spanNulls": true,
5467                 "stacking": {
5468                   "group": "A",
5469                   "mode": "none"
5470                 },
5471                 "thresholdsStyle": {
5472                   "mode": "off"
5473                 }
5474               },
5475               "links": [],
5476               "mappings": [],
5477               "thresholds": {
5478                 "mode": "absolute",
5479                 "steps": [
5480                   {
5481                     "color": "green"
5482                   },
5483                   {
5484                     "color": "red",
5485                     "value": 80
5486                   }
5487                 ]
5488               },
5489               "unit": "short"
5490             },
5491             "overrides": []
5492           },
5493           "gridPos": {
5494             "h": 8,
5495             "w": 12,
5496             "x": 12,
5497             "y": 40
5498           },
5499           "id": 106,
5500           "options": {
5501             "legend": {
5502               "calcs": [
5503                 "mean",
5504                 "lastNotNull",
5505                 "max"
5506               ],
5507               "displayMode": "table",
5508               "placement": "bottom",
5509               "showLegend": true
5510             },
5511             "tooltip": {
5512               "mode": "single"
5513             }
5514           },
5515           "pluginVersion": "8.1.3",
5516           "targets": [
5517             {
5518               "datasource": {
5519                 "type": "prometheus",
5520                 "uid": "PBFA97CFB590B2093"
5521               },
5522               "expr": "sum(kafka_server_socketservermetrics_connection_creation_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5523               "interval": "",
5524               "legendFormat": "{{instance}}",
5525               "refId": "A"
5526             }
5527           ],
5528           "title": "Connections creation rate per instance",
5529           "type": "timeseries"
5530         },
5531         {
5532           "datasource": {
5533             "type": "prometheus",
5534             "uid": "PBFA97CFB590B2093"
5535           },
5536           "fieldConfig": {
5537             "defaults": {
5538               "color": {
5539                 "mode": "palette-classic"
5540               },
5541               "custom": {
5542                 "axisLabel": "",
5543                 "axisPlacement": "auto",
5544                 "barAlignment": 0,
5545                 "drawStyle": "line",
5546                 "fillOpacity": 10,
5547                 "gradientMode": "none",
5548                 "hideFrom": {
5549                   "legend": false,
5550                   "tooltip": false,
5551                   "viz": false
5552                 },
5553                 "lineInterpolation": "linear",
5554                 "lineWidth": 1,
5555                 "pointSize": 5,
5556                 "scaleDistribution": {
5557                   "type": "linear"
5558                 },
5559                 "showPoints": "never",
5560                 "spanNulls": true,
5561                 "stacking": {
5562                   "group": "A",
5563                   "mode": "none"
5564                 },
5565                 "thresholdsStyle": {
5566                   "mode": "off"
5567                 }
5568               },
5569               "links": [],
5570               "mappings": [],
5571               "thresholds": {
5572                 "mode": "absolute",
5573                 "steps": [
5574                   {
5575                     "color": "green"
5576                   },
5577                   {
5578                     "color": "red",
5579                     "value": 80
5580                   }
5581                 ]
5582               },
5583               "unit": "short"
5584             },
5585             "overrides": []
5586           },
5587           "gridPos": {
5588             "h": 8,
5589             "w": 12,
5590             "x": 0,
5591             "y": 48
5592           },
5593           "id": 108,
5594           "options": {
5595             "legend": {
5596               "calcs": [
5597                 "mean",
5598                 "lastNotNull",
5599                 "max"
5600               ],
5601               "displayMode": "table",
5602               "placement": "bottom",
5603               "showLegend": true
5604             },
5605             "tooltip": {
5606               "mode": "single"
5607             }
5608           },
5609           "pluginVersion": "8.1.3",
5610           "targets": [
5611             {
5612               "datasource": {
5613                 "type": "prometheus",
5614                 "uid": "PBFA97CFB590B2093"
5615               },
5616               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (listener)",
5617               "interval": "",
5618               "legendFormat": "{{listener}}",
5619               "refId": "A"
5620             }
5621           ],
5622           "title": "Connections close rate per listener",
5623           "type": "timeseries"
5624         },
5625         {
5626           "datasource": {
5627             "type": "prometheus",
5628             "uid": "PBFA97CFB590B2093"
5629           },
5630           "fieldConfig": {
5631             "defaults": {
5632               "color": {
5633                 "mode": "palette-classic"
5634               },
5635               "custom": {
5636                 "axisLabel": "",
5637                 "axisPlacement": "auto",
5638                 "barAlignment": 0,
5639                 "drawStyle": "line",
5640                 "fillOpacity": 10,
5641                 "gradientMode": "none",
5642                 "hideFrom": {
5643                   "legend": false,
5644                   "tooltip": false,
5645                   "viz": false
5646                 },
5647                 "lineInterpolation": "linear",
5648                 "lineWidth": 1,
5649                 "pointSize": 5,
5650                 "scaleDistribution": {
5651                   "type": "linear"
5652                 },
5653                 "showPoints": "never",
5654                 "spanNulls": true,
5655                 "stacking": {
5656                   "group": "A",
5657                   "mode": "none"
5658                 },
5659                 "thresholdsStyle": {
5660                   "mode": "off"
5661                 }
5662               },
5663               "links": [],
5664               "mappings": [],
5665               "thresholds": {
5666                 "mode": "absolute",
5667                 "steps": [
5668                   {
5669                     "color": "green"
5670                   },
5671                   {
5672                     "color": "red",
5673                     "value": 80
5674                   }
5675                 ]
5676               },
5677               "unit": "short"
5678             },
5679             "overrides": []
5680           },
5681           "gridPos": {
5682             "h": 8,
5683             "w": 12,
5684             "x": 12,
5685             "y": 48
5686           },
5687           "id": 110,
5688           "options": {
5689             "legend": {
5690               "calcs": [
5691                 "mean",
5692                 "lastNotNull",
5693                 "max"
5694               ],
5695               "displayMode": "table",
5696               "placement": "bottom",
5697               "showLegend": true
5698             },
5699             "tooltip": {
5700               "mode": "single"
5701             }
5702           },
5703           "pluginVersion": "8.1.3",
5704           "targets": [
5705             {
5706               "datasource": {
5707                 "type": "prometheus",
5708                 "uid": "PBFA97CFB590B2093"
5709               },
5710               "expr": "sum(kafka_server_socketservermetrics_connection_close_rate{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (instance)",
5711               "interval": "",
5712               "legendFormat": "{{instance}}",
5713               "refId": "A"
5714             }
5715           ],
5716           "title": "Connections close rate per instance",
5717           "type": "timeseries"
5718         },
5719         {
5720           "datasource": {
5721             "type": "prometheus",
5722             "uid": "PBFA97CFB590B2093"
5723           },
5724           "description": "Tracks the amount of time Acceptor is blocked from accepting connections. See KIP-402 for more details.",
5725           "fieldConfig": {
5726             "defaults": {
5727               "color": {
5728                 "mode": "palette-classic"
5729               },
5730               "custom": {
5731                 "axisLabel": "",
5732                 "axisPlacement": "auto",
5733                 "barAlignment": 0,
5734                 "drawStyle": "line",
5735                 "fillOpacity": 10,
5736                 "gradientMode": "none",
5737                 "hideFrom": {
5738                   "legend": false,
5739                   "tooltip": false,
5740                   "viz": false
5741                 },
5742                 "lineInterpolation": "linear",
5743                 "lineWidth": 1,
5744                 "pointSize": 5,
5745                 "scaleDistribution": {
5746                   "type": "linear"
5747                 },
5748                 "showPoints": "never",
5749                 "spanNulls": true,
5750                 "stacking": {
5751                   "group": "A",
5752                   "mode": "none"
5753                 },
5754                 "thresholdsStyle": {
5755                   "mode": "off"
5756                 }
5757               },
5758               "links": [],
5759               "mappings": [],
5760               "min": 0,
5761               "thresholds": {
5762                 "mode": "absolute",
5763                 "steps": [
5764                   {
5765                     "color": "green"
5766                   },
5767                   {
5768                     "color": "red",
5769                     "value": 80
5770                   }
5771                 ]
5772               },
5773               "unit": "percent"
5774             },
5775             "overrides": []
5776           },
5777           "gridPos": {
5778             "h": 8,
5779             "w": 12,
5780             "x": 0,
5781             "y": 56
5782           },
5783           "id": 112,
5784           "options": {
5785             "legend": {
5786               "calcs": [
5787                 "mean",
5788                 "lastNotNull",
5789                 "max"
5790               ],
5791               "displayMode": "table",
5792               "placement": "bottom",
5793               "showLegend": true
5794             },
5795             "tooltip": {
5796               "mode": "single"
5797             }
5798           },
5799           "pluginVersion": "8.1.3",
5800           "targets": [
5801             {
5802               "datasource": {
5803                 "type": "prometheus",
5804                 "uid": "PBFA97CFB590B2093"
5805               },
5806               "expr": "kafka_network_acceptor_acceptorblockedpercent{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
5807               "interval": "",
5808               "legendFormat": "{{instance}} - {{listener}}",
5809               "refId": "A"
5810             }
5811           ],
5812           "title": "Acceptor Blocked Percentage",
5813           "type": "timeseries"
5814         },
5815         {
5816           "datasource": {
5817             "type": "prometheus",
5818             "uid": "PBFA97CFB590B2093"
5819           },
5820           "fieldConfig": {
5821             "defaults": {
5822               "color": {
5823                 "mode": "palette-classic"
5824               },
5825               "custom": {
5826                 "axisLabel": "",
5827                 "axisPlacement": "auto",
5828                 "barAlignment": 0,
5829                 "drawStyle": "line",
5830                 "fillOpacity": 10,
5831                 "gradientMode": "none",
5832                 "hideFrom": {
5833                   "legend": false,
5834                   "tooltip": false,
5835                   "viz": false
5836                 },
5837                 "lineInterpolation": "linear",
5838                 "lineWidth": 1,
5839                 "pointSize": 5,
5840                 "scaleDistribution": {
5841                   "type": "linear"
5842                 },
5843                 "showPoints": "never",
5844                 "spanNulls": true,
5845                 "stacking": {
5846                   "group": "A",
5847                   "mode": "none"
5848                 },
5849                 "thresholdsStyle": {
5850                   "mode": "off"
5851                 }
5852               },
5853               "links": [],
5854               "mappings": [],
5855               "thresholds": {
5856                 "mode": "absolute",
5857                 "steps": [
5858                   {
5859                     "color": "green"
5860                   },
5861                   {
5862                     "color": "red",
5863                     "value": 80
5864                   }
5865                 ]
5866               },
5867               "unit": "short"
5868             },
5869             "overrides": []
5870           },
5871           "gridPos": {
5872             "h": 8,
5873             "w": 12,
5874             "x": 12,
5875             "y": 56
5876           },
5877           "id": 114,
5878           "options": {
5879             "legend": {
5880               "calcs": [
5881                 "mean",
5882                 "lastNotNull",
5883                 "max"
5884               ],
5885               "displayMode": "table",
5886               "placement": "bottom",
5887               "showLegend": true
5888             },
5889             "tooltip": {
5890               "mode": "single"
5891             }
5892           },
5893           "pluginVersion": "8.1.3",
5894           "targets": [
5895             {
5896               "datasource": {
5897                 "type": "prometheus",
5898                 "uid": "PBFA97CFB590B2093"
5899               },
5900               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version)",
5901               "interval": "",
5902               "legendFormat": "{{client_software_name}} {{client_software_version}}",
5903               "refId": "A"
5904             }
5905           ],
5906           "title": "Connections per client version",
5907           "type": "timeseries"
5908         }
5909       ],
5910       "targets": [
5911         {
5912           "datasource": {
5913             "type": "datasource",
5914             "uid": "grafana"
5915           },
5916           "refId": "A"
5917         }
5918       ],
5919       "title": "Connections",
5920       "type": "row"
5921     },
5922     {
5923       "collapsed": true,
5924       "datasource": {
5925         "type": "datasource",
5926         "uid": "grafana"
5927       },
5928       "gridPos": {
5929         "h": 1,
5930         "w": 24,
5931         "x": 0,
5932         "y": 54
5933       },
5934       "id": 120,
5935       "panels": [
5936         {
5937           "datasource": {
5938             "type": "prometheus",
5939             "uid": "PBFA97CFB590B2093"
5940           },
5941           "description": "Number of consumer groups per group coordinator",
5942           "fieldConfig": {
5943             "defaults": {
5944               "color": {
5945                 "mode": "palette-classic"
5946               },
5947               "custom": {
5948                 "axisLabel": "",
5949                 "axisPlacement": "auto",
5950                 "barAlignment": 0,
5951                 "drawStyle": "line",
5952                 "fillOpacity": 10,
5953                 "gradientMode": "none",
5954                 "hideFrom": {
5955                   "legend": false,
5956                   "tooltip": false,
5957                   "viz": false
5958                 },
5959                 "lineInterpolation": "linear",
5960                 "lineWidth": 1,
5961                 "pointSize": 5,
5962                 "scaleDistribution": {
5963                   "type": "linear"
5964                 },
5965                 "showPoints": "never",
5966                 "spanNulls": true,
5967                 "stacking": {
5968                   "group": "A",
5969                   "mode": "none"
5970                 },
5971                 "thresholdsStyle": {
5972                   "mode": "off"
5973                 }
5974               },
5975               "links": [],
5976               "mappings": [],
5977               "thresholds": {
5978                 "mode": "absolute",
5979                 "steps": [
5980                   {
5981                     "color": "green"
5982                   },
5983                   {
5984                     "color": "red",
5985                     "value": 80
5986                   }
5987                 ]
5988               },
5989               "unit": "short"
5990             },
5991             "overrides": []
5992           },
5993           "gridPos": {
5994             "h": 8,
5995             "w": 12,
5996             "x": 0,
5997             "y": 33
5998           },
5999           "id": 116,
6000           "options": {
6001             "legend": {
6002               "calcs": [
6003                 "mean",
6004                 "lastNotNull",
6005                 "max"
6006               ],
6007               "displayMode": "table",
6008               "placement": "bottom",
6009               "showLegend": true
6010             },
6011             "tooltip": {
6012               "mode": "single"
6013             }
6014           },
6015           "pluginVersion": "8.1.3",
6016           "targets": [
6017             {
6018               "datasource": {
6019                 "type": "prometheus",
6020                 "uid": "PBFA97CFB590B2093"
6021               },
6022               "expr": "kafka_coordinator_group_groupmetadatamanager_numgroups{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}",
6023               "instant": false,
6024               "interval": "",
6025               "legendFormat": "{{instance}}",
6026               "refId": "A"
6027             }
6028           ],
6029           "title": "Consumer groups number per coordinator",
6030           "type": "timeseries"
6031         },
6032         {
6033           "datasource": {
6034             "type": "prometheus",
6035             "uid": "PBFA97CFB590B2093"
6036           },
6037           "description": "Number of consumer group per state",
6038           "fieldConfig": {
6039             "defaults": {
6040               "color": {
6041                 "mode": "palette-classic"
6042               },
6043               "custom": {
6044                 "axisLabel": "",
6045                 "axisPlacement": "auto",
6046                 "barAlignment": 0,
6047                 "drawStyle": "line",
6048                 "fillOpacity": 10,
6049                 "gradientMode": "none",
6050                 "hideFrom": {
6051                   "legend": false,
6052                   "tooltip": false,
6053                   "viz": false
6054                 },
6055                 "lineInterpolation": "linear",
6056                 "lineWidth": 1,
6057                 "pointSize": 5,
6058                 "scaleDistribution": {
6059                   "type": "linear"
6060                 },
6061                 "showPoints": "never",
6062                 "spanNulls": true,
6063                 "stacking": {
6064                   "group": "A",
6065                   "mode": "none"
6066                 },
6067                 "thresholdsStyle": {
6068                   "mode": "off"
6069                 }
6070               },
6071               "links": [],
6072               "mappings": [],
6073               "thresholds": {
6074                 "mode": "absolute",
6075                 "steps": [
6076                   {
6077                     "color": "green"
6078                   },
6079                   {
6080                     "color": "red",
6081                     "value": 80
6082                   }
6083                 ]
6084               },
6085               "unit": "short"
6086             },
6087             "overrides": []
6088           },
6089           "gridPos": {
6090             "h": 8,
6091             "w": 12,
6092             "x": 12,
6093             "y": 33
6094           },
6095           "id": 118,
6096           "options": {
6097             "legend": {
6098               "calcs": [
6099                 "mean",
6100                 "lastNotNull",
6101                 "max"
6102               ],
6103               "displayMode": "table",
6104               "placement": "bottom",
6105               "showLegend": true
6106             },
6107             "tooltip": {
6108               "mode": "single"
6109             }
6110           },
6111           "pluginVersion": "8.1.3",
6112           "targets": [
6113             {
6114               "datasource": {
6115                 "type": "prometheus",
6116                 "uid": "PBFA97CFB590B2093"
6117               },
6118               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsstable{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6119               "instant": false,
6120               "interval": "",
6121               "legendFormat": "stable",
6122               "refId": "A"
6123             },
6124             {
6125               "datasource": {
6126                 "type": "prometheus",
6127                 "uid": "PBFA97CFB590B2093"
6128               },
6129               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupspreparingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6130               "interval": "",
6131               "legendFormat": "preparing-rebalance",
6132               "refId": "B"
6133             },
6134             {
6135               "datasource": {
6136                 "type": "prometheus",
6137                 "uid": "PBFA97CFB590B2093"
6138               },
6139               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsdead{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6140               "interval": "",
6141               "legendFormat": "dead",
6142               "refId": "C"
6143             },
6144             {
6145               "datasource": {
6146                 "type": "prometheus",
6147                 "uid": "PBFA97CFB590B2093"
6148               },
6149               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupscompletingrebalance{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6150               "interval": "",
6151               "legendFormat": "completing-rebalance",
6152               "refId": "D"
6153             },
6154             {
6155               "datasource": {
6156                 "type": "prometheus",
6157                 "uid": "PBFA97CFB590B2093"
6158               },
6159               "expr": "sum(kafka_coordinator_group_groupmetadatamanager_numgroupsempty{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"})",
6160               "interval": "",
6161               "legendFormat": "empty",
6162               "refId": "E"
6163             }
6164           ],
6165           "title": "Nb consumer groups per state",
6166           "type": "timeseries"
6167         }
6168       ],
6169       "targets": [
6170         {
6171           "datasource": {
6172             "type": "datasource",
6173             "uid": "grafana"
6174           },
6175           "refId": "A"
6176         }
6177       ],
6178       "title": "Group Coordinator",
6179       "type": "row"
6180     },
6181     {
6182       "collapsed": true,
6183       "datasource": {
6184         "type": "datasource",
6185         "uid": "grafana"
6186       },
6187       "gridPos": {
6188         "h": 1,
6189         "w": 24,
6190         "x": 0,
6191         "y": 55
6192       },
6193       "id": 46,
6194       "panels": [
6195         {
6196           "datasource": {
6197             "type": "prometheus",
6198             "uid": "PBFA97CFB590B2093"
6199           },
6200           "description": "The number of messages produced converted to match the log.message.format.version.",
6201           "fieldConfig": {
6202             "defaults": {
6203               "color": {
6204                 "mode": "palette-classic"
6205               },
6206               "custom": {
6207                 "axisLabel": "",
6208                 "axisPlacement": "auto",
6209                 "barAlignment": 0,
6210                 "drawStyle": "line",
6211                 "fillOpacity": 10,
6212                 "gradientMode": "none",
6213                 "hideFrom": {
6214                   "legend": false,
6215                   "tooltip": false,
6216                   "viz": false
6217                 },
6218                 "lineInterpolation": "linear",
6219                 "lineWidth": 1,
6220                 "pointSize": 5,
6221                 "scaleDistribution": {
6222                   "type": "linear"
6223                 },
6224                 "showPoints": "never",
6225                 "spanNulls": true,
6226                 "stacking": {
6227                   "group": "A",
6228                   "mode": "none"
6229                 },
6230                 "thresholdsStyle": {
6231                   "mode": "off"
6232                 }
6233               },
6234               "links": [],
6235               "mappings": [],
6236               "min": 0,
6237               "thresholds": {
6238                 "mode": "absolute",
6239                 "steps": [
6240                   {
6241                     "color": "green"
6242                   },
6243                   {
6244                     "color": "red",
6245                     "value": 80
6246                   }
6247                 ]
6248               },
6249               "unit": "ops"
6250             },
6251             "overrides": []
6252           },
6253           "gridPos": {
6254             "h": 8,
6255             "w": 8,
6256             "x": 0,
6257             "y": 34
6258           },
6259           "id": 48,
6260           "options": {
6261             "legend": {
6262               "calcs": [
6263                 "mean",
6264                 "lastNotNull",
6265                 "max"
6266               ],
6267               "displayMode": "list",
6268               "placement": "bottom",
6269               "showLegend": true
6270             },
6271             "tooltip": {
6272               "mode": "single"
6273             }
6274           },
6275           "pluginVersion": "8.1.3",
6276           "targets": [
6277             {
6278               "datasource": {
6279                 "type": "prometheus",
6280                 "uid": "PBFA97CFB590B2093"
6281               },
6282               "expr": "sum(kafka_server_brokertopicmetrics_producemessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6283               "hide": false,
6284               "interval": "",
6285               "legendFormat": "{{topic}}",
6286               "refId": "A"
6287             }
6288           ],
6289           "title": "Number of produced message conversion",
6290           "type": "timeseries"
6291         },
6292         {
6293           "datasource": {
6294             "type": "prometheus",
6295             "uid": "PBFA97CFB590B2093"
6296           },
6297           "description": "The number of messages consumed converted at consumer to match the log.message.format.version.",
6298           "fieldConfig": {
6299             "defaults": {
6300               "color": {
6301                 "mode": "palette-classic"
6302               },
6303               "custom": {
6304                 "axisLabel": "",
6305                 "axisPlacement": "auto",
6306                 "barAlignment": 0,
6307                 "drawStyle": "line",
6308                 "fillOpacity": 10,
6309                 "gradientMode": "none",
6310                 "hideFrom": {
6311                   "legend": false,
6312                   "tooltip": false,
6313                   "viz": false
6314                 },
6315                 "lineInterpolation": "linear",
6316                 "lineWidth": 1,
6317                 "pointSize": 5,
6318                 "scaleDistribution": {
6319                   "type": "linear"
6320                 },
6321                 "showPoints": "never",
6322                 "spanNulls": true,
6323                 "stacking": {
6324                   "group": "A",
6325                   "mode": "none"
6326                 },
6327                 "thresholdsStyle": {
6328                   "mode": "off"
6329                 }
6330               },
6331               "links": [],
6332               "mappings": [],
6333               "min": 0,
6334               "thresholds": {
6335                 "mode": "absolute",
6336                 "steps": [
6337                   {
6338                     "color": "green"
6339                   },
6340                   {
6341                     "color": "red",
6342                     "value": 80
6343                   }
6344                 ]
6345               },
6346               "unit": "ops"
6347             },
6348             "overrides": []
6349           },
6350           "gridPos": {
6351             "h": 8,
6352             "w": 8,
6353             "x": 8,
6354             "y": 34
6355           },
6356           "id": 51,
6357           "options": {
6358             "legend": {
6359               "calcs": [
6360                 "mean",
6361                 "lastNotNull",
6362                 "max"
6363               ],
6364               "displayMode": "list",
6365               "placement": "bottom",
6366               "showLegend": true
6367             },
6368             "tooltip": {
6369               "mode": "single"
6370             }
6371           },
6372           "pluginVersion": "8.1.3",
6373           "targets": [
6374             {
6375               "datasource": {
6376                 "type": "prometheus",
6377                 "uid": "PBFA97CFB590B2093"
6378               },
6379               "expr": "sum(kafka_server_brokertopicmetrics_fetchmessageconversionspersec{job=\"kafka-broker\",env=\"$env\",instance=~\"$instance\"})",
6380               "hide": false,
6381               "interval": "",
6382               "legendFormat": "{{topic}}",
6383               "refId": "A"
6384             }
6385           ],
6386           "title": "Number of consumed message conversion",
6387           "type": "timeseries"
6388         },
6389         {
6390           "datasource": {
6391             "type": "datasource",
6392             "uid": "grafana"
6393           },
6394           "description": "Number of connection per client version",
6395           "fieldConfig": {
6396             "defaults": {
6397               "color": {
6398                 "mode": "palette-classic"
6399               },
6400               "custom": {
6401                 "hideFrom": {
6402                   "legend": false,
6403                   "tooltip": false,
6404                   "viz": false
6405                 }
6406               },
6407               "decimals": 0,
6408               "mappings": [],
6409               "unit": "short"
6410             },
6411             "overrides": []
6412           },
6413           "gridPos": {
6414             "h": 8,
6415             "w": 8,
6416             "x": 16,
6417             "y": 34
6418           },
6419           "id": 96,
6420           "links": [],
6421           "options": {
6422             "legend": {
6423               "calcs": [],
6424               "displayMode": "table",
6425               "placement": "bottom",
6426               "showLegend": true,
6427               "values": [
6428                 "value"
6429               ]
6430             },
6431             "pieType": "pie",
6432             "reduceOptions": {
6433               "calcs": [
6434                 "lastNotNull"
6435               ],
6436               "fields": "",
6437               "values": false
6438             },
6439             "tooltip": {
6440               "mode": "single"
6441             }
6442           },
6443           "targets": [
6444             {
6445               "datasource": {
6446                 "type": "datasource",
6447                 "uid": "grafana"
6448               },
6449               "expr": "sum(kafka_server_socketservermetrics_connections{job=\"kafka-broker\", env=\"$env\", instance=~\"$instance\"}) by (client_software_name, client_software_version) ",
6450               "interval": "",
6451               "legendFormat": "{{client_software_name}} - {{client_software_version}}",
6452               "refId": "A"
6453             }
6454           ],
6455           "title": "Client version repartition",
6456           "type": "piechart"
6457         }
6458       ],
6459       "targets": [
6460         {
6461           "datasource": {
6462             "type": "datasource",
6463             "uid": "grafana"
6464           },
6465           "refId": "A"
6466         }
6467       ],
6468       "title": "Message Conversion",
6469       "type": "row"
6470     }
6471   ],
6472   "refresh": "5s",
6473   "schemaVersion": 38,
6474   "style": "dark",
6475   "tags": [],
6476   "templating": {
6477     "list": [
6478       {
6479         "current": {
6480           "selected": false,
6481           "text": "dev",
6482           "value": "dev"
6483         },
6484         "datasource": {
6485           "type": "prometheus",
6486           "uid": "PBFA97CFB590B2093"
6487         },
6488         "definition": "label_values(env)",
6489         "hide": 0,
6490         "includeAll": false,
6491         "label": "Environment",
6492         "multi": false,
6493         "name": "env",
6494         "options": [],
6495         "query": {
6496           "query": "label_values(env)",
6497           "refId": "Prometheus-env-Variable-Query"
6498         },
6499         "refresh": 1,
6500         "regex": "",
6501         "skipUrlSync": false,
6502         "sort": 0,
6503         "tagValuesQuery": "",
6504         "tagsQuery": "",
6505         "type": "query",
6506         "useTags": false
6507       },
6508       {
6509         "current": {
6510           "selected": true,
6511           "text": [
6512             "All"
6513           ],
6514           "value": [
6515             "$__all"
6516           ]
6517         },
6518         "datasource": {
6519           "type": "prometheus",
6520           "uid": "PBFA97CFB590B2093"
6521         },
6522         "definition": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6523         "hide": 0,
6524         "includeAll": true,
6525         "label": "Instance",
6526         "multi": true,
6527         "name": "instance",
6528         "options": [],
6529         "query": {
6530           "query": "label_values(kafka_server_kafkaserver_brokerstate{env=\"${env}\"}, instance)",
6531           "refId": "Prometheus-instance-Variable-Query"
6532         },
6533         "refresh": 1,
6534         "regex": "",
6535         "skipUrlSync": false,
6536         "sort": 0,
6537         "tagValuesQuery": "",
6538         "tagsQuery": "",
6539         "type": "query",
6540         "useTags": false
6541       },
6542       {
6543         "allValue": "",
6544         "current": {
6545           "selected": true,
6546           "text": [
6547             "0.95"
6548           ],
6549           "value": [
6550             "0.95"
6551           ]
6552         },
6553         "datasource": {
6554           "type": "prometheus",
6555           "uid": "PBFA97CFB590B2093"
6556         },
6557         "definition": "label_values(quantile)",
6558         "hide": 0,
6559         "includeAll": true,
6560         "label": "Percentile",
6561         "multi": true,
6562         "name": "percentile",
6563         "options": [],
6564         "query": {
6565           "query": "label_values(quantile)",
6566           "refId": "Prometheus-percentile-Variable-Query"
6567         },
6568         "refresh": 1,
6569         "regex": "",
6570         "skipUrlSync": false,
6571         "sort": 0,
6572         "tagValuesQuery": "",
6573         "tagsQuery": "",
6574         "type": "query",
6575         "useTags": false
6576       }
6577     ]
6578   },
6579   "time": {
6580     "from": "now-12h",
6581     "to": "now"
6582   },
6583   "timepicker": {
6584     "refresh_intervals": [
6585       "5s",
6586       "10s",
6587       "30s",
6588       "1m",
6589       "5m",
6590       "15m",
6591       "30m",
6592       "1h",
6593       "2h",
6594       "1d"
6595     ],
6596     "time_options": [
6597       "5m",
6598       "15m",
6599       "1h",
6600       "6h",
6601       "12h",
6602       "24h",
6603       "2d",
6604       "7d",
6605       "30d"
6606     ]
6607   },
6608   "timezone": "browser",
6609   "title": "Kafka cluster",
6610   "uid": "qu-QZdfZz",
6611   "version": 1,
6612   "weekStart": ""
6613 }