juplo.de Git - demos/kafka/monitoring/blob - Kafka_Lag_Exporter_Dashboard.json

   1 {
   2   "__inputs": [
   3     {
   4       "name": "DS_CINNAMON_PROMETHEUS",
   5       "label": "Cinnamon Prometheus",
   6       "description": "",
   7       "type": "datasource",
   8       "pluginId": "prometheus",
   9       "pluginName": "Prometheus"
  10     }
  11   ],
  12   "__requires": [
  13     {
  14       "type": "grafana",
  15       "id": "grafana",
  16       "name": "Grafana",
  17       "version": "5.3.4"
  18     },
  19     {
  20       "type": "panel",
  21       "id": "graph",
  22       "name": "Graph",
  23       "version": "5.0.0"
  24     },
  25     {
  26       "type": "datasource",
  27       "id": "prometheus",
  28       "name": "Prometheus",
  29       "version": "5.0.0"
  30     }
  31   ],
  32   "annotations": {
  33     "list": [
  34       {
  35         "builtIn": 1,
  36         "datasource": "-- Grafana --",
  37         "enable": true,
  38         "hide": true,
  39         "iconColor": "rgba(0, 211, 255, 1)",
  40         "name": "Annotations & Alerts",
  41         "type": "dashboard"
  42       }
  43     ]
  44   },
  45   "editable": true,
  46   "gnetId": null,
  47   "graphTooltip": 0,
  48   "id": null,
  49   "iteration": 1556312150750,
  50   "links": [],
  51   "panels": [
  52     {
  53       "collapsed": false,
  54       "gridPos": {
  55         "h": 1,
  56         "w": 24,
  57         "x": 0,
  58         "y": 0
  59       },
  60       "id": 28,
  61       "panels": [],
  62       "repeat": null,
  63       "title": "All Consumer Group Lag",
  64       "type": "row"
  65     },
  66     {
  67       "aliasColors": {},
  68       "bars": false,
  69       "dashLength": 10,
  70       "dashes": false,
  71       "datasource": "${DS_CINNAMON_PROMETHEUS}",
  72       "description": "Max extrapolated lag in seconds for each consumer group.",
  73       "fill": 1,
  74       "gridPos": {
  75         "h": 10,
  76         "w": 12,
  77         "x": 0,
  78         "y": 1
  79       },
  80       "id": 102,
  81       "legend": {
  82         "avg": false,
  83         "current": false,
  84         "max": false,
  85         "min": false,
  86         "show": true,
  87         "total": false,
  88         "values": false
  89       },
  90       "lines": true,
  91       "linewidth": 1,
  92       "links": [],
  93       "nullPointMode": "null",
  94       "percentage": false,
  95       "pointradius": 5,
  96       "points": false,
  97       "renderer": "flot",
  98       "seriesOverrides": [],
  99       "spaceLength": 10,
 100       "stack": false,
 101       "steppedLine": false,
 102       "targets": [
 103         {
 104           "expr": "topk(50, kafka_consumergroup_group_max_lag_seconds{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"})",
 105           "format": "time_series",
 106           "hide": false,
 107           "intervalFactor": 1,
 108           "legendFormat": "{{group}}",
 109           "refId": "A"
 110         }
 111       ],
 112       "thresholds": [],
 113       "timeFrom": null,
 114       "timeShift": null,
 115       "title": "Consumer Group Max Lag Seconds",
 116       "tooltip": {
 117         "shared": true,
 118         "sort": 0,
 119         "value_type": "individual"
 120       },
 121       "type": "graph",
 122       "xaxis": {
 123         "buckets": null,
 124         "mode": "time",
 125         "name": null,
 126         "show": true,
 127         "values": []
 128       },
 129       "yaxes": [
 130         {
 131           "format": "s",
 132           "label": "estimated lag in seconds",
 133           "logBase": 1,
 134           "max": null,
 135           "min": "0",
 136           "show": true
 137         },
 138         {
 139           "format": "short",
 140           "label": "",
 141           "logBase": 1,
 142           "max": null,
 143           "min": null,
 144           "show": false
 145         }
 146       ],
 147       "yaxis": {
 148         "align": false,
 149         "alignLevel": null
 150       }
 151     },
 152     {
 153       "aliasColors": {},
 154       "bars": false,
 155       "dashLength": 10,
 156       "dashes": false,
 157       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 158       "description": "Extrapolated lag in seconds for each partition.",
 159       "fill": 1,
 160       "gridPos": {
 161         "h": 10,
 162         "w": 12,
 163         "x": 12,
 164         "y": 1
 165       },
 166       "id": 103,
 167       "legend": {
 168         "avg": false,
 169         "current": false,
 170         "max": false,
 171         "min": false,
 172         "show": true,
 173         "total": false,
 174         "values": false
 175       },
 176       "lines": true,
 177       "linewidth": 1,
 178       "links": [],
 179       "nullPointMode": "null",
 180       "percentage": false,
 181       "pointradius": 5,
 182       "points": false,
 183       "renderer": "flot",
 184       "seriesOverrides": [],
 185       "spaceLength": 10,
 186       "stack": false,
 187       "steppedLine": false,
 188       "targets": [
 189         {
 190           "expr": "topk(25, kafka_consumergroup_group_lag_seconds{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"})",
 191           "format": "time_series",
 192           "hide": false,
 193           "intervalFactor": 1,
 194           "legendFormat": "{{group}},{{topic}},{{partition}}",
 195           "refId": "A"
 196         }
 197       ],
 198       "thresholds": [],
 199       "timeFrom": null,
 200       "timeShift": null,
 201       "title": "Consumer Group Lag Partition Seconds",
 202       "tooltip": {
 203         "shared": true,
 204         "sort": 0,
 205         "value_type": "individual"
 206       },
 207       "type": "graph",
 208       "xaxis": {
 209         "buckets": null,
 210         "mode": "time",
 211         "name": null,
 212         "show": true,
 213         "values": []
 214       },
 215       "yaxes": [
 216         {
 217           "format": "s",
 218           "label": "estimated lag in seconds",
 219           "logBase": 1,
 220           "max": null,
 221           "min": "0",
 222           "show": true
 223         },
 224         {
 225           "format": "short",
 226           "label": "",
 227           "logBase": 1,
 228           "max": null,
 229           "min": null,
 230           "show": false
 231         }
 232       ],
 233       "yaxis": {
 234         "align": false,
 235         "alignLevel": null
 236       }
 237     },
 238     {
 239       "aliasColors": {},
 240       "bars": false,
 241       "dashLength": 10,
 242       "dashes": false,
 243       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 244       "description": "Max offset lag for each consumer group.",
 245       "fill": 1,
 246       "gridPos": {
 247         "h": 10,
 248         "w": 12,
 249         "x": 0,
 250         "y": 11
 251       },
 252       "id": 98,
 253       "legend": {
 254         "avg": false,
 255         "current": false,
 256         "max": false,
 257         "min": false,
 258         "show": true,
 259         "total": false,
 260         "values": false
 261       },
 262       "lines": true,
 263       "linewidth": 1,
 264       "links": [],
 265       "nullPointMode": "null",
 266       "percentage": false,
 267       "pointradius": 5,
 268       "points": false,
 269       "renderer": "flot",
 270       "repeat": null,
 271       "repeatDirection": "h",
 272       "seriesOverrides": [],
 273       "spaceLength": 10,
 274       "stack": false,
 275       "steppedLine": false,
 276       "targets": [
 277         {
 278           "expr": "topk(50, kafka_consumergroup_group_max_lag{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"})",
 279           "format": "time_series",
 280           "hide": false,
 281           "intervalFactor": 1,
 282           "legendFormat": "{{group}}",
 283           "refId": "A"
 284         }
 285       ],
 286       "thresholds": [],
 287       "timeFrom": null,
 288       "timeShift": null,
 289       "title": "Consumer Group Max Lag Offsets",
 290       "tooltip": {
 291         "shared": true,
 292         "sort": 0,
 293         "value_type": "individual"
 294       },
 295       "type": "graph",
 296       "xaxis": {
 297         "buckets": null,
 298         "mode": "time",
 299         "name": null,
 300         "show": true,
 301         "values": []
 302       },
 303       "yaxes": [
 304         {
 305           "format": "none",
 306           "label": "offsets",
 307           "logBase": 1,
 308           "max": null,
 309           "min": "0",
 310           "show": true
 311         },
 312         {
 313           "format": "short",
 314           "label": "",
 315           "logBase": 1,
 316           "max": null,
 317           "min": null,
 318           "show": false
 319         }
 320       ],
 321       "yaxis": {
 322         "align": false,
 323         "alignLevel": null
 324       }
 325     },
 326     {
 327       "aliasColors": {},
 328       "bars": false,
 329       "dashLength": 10,
 330       "dashes": false,
 331       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 332       "description": "Consumer Group Lag Partition Offsets",
 333       "fill": 1,
 334       "gridPos": {
 335         "h": 10,
 336         "w": 12,
 337         "x": 12,
 338         "y": 11
 339       },
 340       "id": 82,
 341       "legend": {
 342         "avg": false,
 343         "current": false,
 344         "max": false,
 345         "min": false,
 346         "show": true,
 347         "total": false,
 348         "values": false
 349       },
 350       "lines": true,
 351       "linewidth": 1,
 352       "links": [],
 353       "nullPointMode": "null",
 354       "percentage": false,
 355       "pointradius": 5,
 356       "points": false,
 357       "renderer": "flot",
 358       "seriesOverrides": [],
 359       "spaceLength": 10,
 360       "stack": false,
 361       "steppedLine": false,
 362       "targets": [
 363         {
 364           "expr": "topk(25, kafka_consumergroup_group_lag{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"})",
 365           "format": "time_series",
 366           "hide": false,
 367           "intervalFactor": 1,
 368           "legendFormat": "{{group}},{{topic}},{{partition}}",
 369           "refId": "A"
 370         }
 371       ],
 372       "thresholds": [],
 373       "timeFrom": null,
 374       "timeShift": null,
 375       "title": "Consumer Group Lag Partition Offsets",
 376       "tooltip": {
 377         "shared": true,
 378         "sort": 0,
 379         "value_type": "individual"
 380       },
 381       "type": "graph",
 382       "xaxis": {
 383         "buckets": null,
 384         "mode": "time",
 385         "name": null,
 386         "show": true,
 387         "values": []
 388       },
 389       "yaxes": [
 390         {
 391           "format": "none",
 392           "label": "offsets",
 393           "logBase": 1,
 394           "max": null,
 395           "min": "0",
 396           "show": true
 397         },
 398         {
 399           "format": "short",
 400           "label": "offsets_2",
 401           "logBase": 1,
 402           "max": null,
 403           "min": null,
 404           "show": false
 405         }
 406       ],
 407       "yaxis": {
 408         "align": false,
 409         "alignLevel": null
 410       }
 411     },
 412     {
 413       "collapsed": false,
 414       "gridPos": {
 415         "h": 1,
 416         "w": 24,
 417         "x": 0,
 418         "y": 21
 419       },
 420       "id": 149,
 421       "panels": [],
 422       "title": "Consumer Group Lag In Time Per Group Over Offset Lag",
 423       "type": "row"
 424     },
 425     {
 426       "aliasColors": {},
 427       "bars": false,
 428       "dashLength": 10,
 429       "dashes": false,
 430       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 431       "description": "",
 432       "fill": 0,
 433       "gridPos": {
 434         "h": 8,
 435         "w": 8,
 436         "x": 0,
 437         "y": 22
 438       },
 439       "id": 147,
 440       "legend": {
 441         "avg": false,
 442         "current": false,
 443         "max": false,
 444         "min": false,
 445         "show": true,
 446         "total": false,
 447         "values": false
 448       },
 449       "lines": true,
 450       "linewidth": 1,
 451       "links": [],
 452       "minSpan": 8,
 453       "nullPointMode": "null",
 454       "percentage": false,
 455       "pointradius": 5,
 456       "points": false,
 457       "renderer": "flot",
 458       "repeat": "consumer_group",
 459       "repeatDirection": "h",
 460       "seriesOverrides": [
 461         {
 462           "alias": "Consumer group lag in offsets",
 463           "color": "#cca300",
 464           "linewidth": 2,
 465           "yaxis": 2
 466         },
 467         {
 468           "alias": "Consumer group lag in time",
 469           "color": "rgb(255, 0, 0)",
 470           "linewidth": 2,
 471           "yaxis": 1
 472         }
 473       ],
 474       "spaceLength": 10,
 475       "stack": false,
 476       "steppedLine": false,
 477       "targets": [
 478         {
 479           "expr": "kafka_consumergroup_group_max_lag_seconds{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"}",
 480           "format": "time_series",
 481           "hide": false,
 482           "intervalFactor": 1,
 483           "legendFormat": "Consumer group lag in time",
 484           "refId": "A"
 485         },
 486         {
 487           "expr": "kafka_consumergroup_group_max_lag{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"}",
 488           "format": "time_series",
 489           "hide": false,
 490           "intervalFactor": 1,
 491           "legendFormat": "Consumer group lag in offsets",
 492           "refId": "B"
 493         }
 494       ],
 495       "thresholds": [],
 496       "timeFrom": null,
 497       "timeShift": null,
 498       "title": "$consumer_group",
 499       "tooltip": {
 500         "shared": true,
 501         "sort": 0,
 502         "value_type": "individual"
 503       },
 504       "transparent": false,
 505       "type": "graph",
 506       "xaxis": {
 507         "buckets": null,
 508         "mode": "time",
 509         "name": null,
 510         "show": true,
 511         "values": []
 512       },
 513       "yaxes": [
 514         {
 515           "format": "s",
 516           "label": "estimated lag in seconds",
 517           "logBase": 1,
 518           "max": null,
 519           "min": "0",
 520           "show": true
 521         },
 522         {
 523           "format": "short",
 524           "label": "lag in offsets",
 525           "logBase": 1,
 526           "max": null,
 527           "min": "0",
 528           "show": true
 529         }
 530       ],
 531       "yaxis": {
 532         "align": false,
 533         "alignLevel": null
 534       }
 535     },
 536     {
 537       "collapsed": false,
 538       "gridPos": {
 539         "h": 1,
 540         "w": 24,
 541         "x": 0,
 542         "y": 30
 543       },
 544       "id": 106,
 545       "panels": [],
 546       "title": "Consumer Group Lag in Time Per Group Over Summed Offsets",
 547       "type": "row"
 548     },
 549     {
 550       "aliasColors": {},
 551       "bars": false,
 552       "dashLength": 10,
 553       "dashes": false,
 554       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 555       "description": "",
 556       "fill": 0,
 557       "gridPos": {
 558         "h": 8,
 559         "w": 8,
 560         "x": 0,
 561         "y": 31
 562       },
 563       "id": 107,
 564       "legend": {
 565         "avg": false,
 566         "current": false,
 567         "max": false,
 568         "min": false,
 569         "show": true,
 570         "total": false,
 571         "values": false
 572       },
 573       "lines": true,
 574       "linewidth": 1,
 575       "links": [],
 576       "minSpan": 8,
 577       "nullPointMode": "null",
 578       "percentage": false,
 579       "pointradius": 5,
 580       "points": false,
 581       "renderer": "flot",
 582       "repeat": "consumer_group",
 583       "repeatDirection": "h",
 584       "seriesOverrides": [
 585         {
 586           "alias": "Sum of group offsets",
 587           "yaxis": 2
 588         },
 589         {
 590           "alias": "Sum of latest offsets",
 591           "yaxis": 2
 592         },
 593         {
 594           "alias": "/Consumer group.*/",
 595           "color": "rgb(255, 0, 0)",
 596           "linewidth": 2
 597         }
 598       ],
 599       "spaceLength": 10,
 600       "stack": false,
 601       "steppedLine": false,
 602       "targets": [
 603         {
 604           "expr": "max(kafka_consumergroup_group_lag_seconds{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"}) by (group)",
 605           "format": "time_series",
 606           "hide": false,
 607           "intervalFactor": 1,
 608           "legendFormat": "Consumer group lag in time",
 609           "refId": "A"
 610         },
 611         {
 612           "expr": "sum(kafka_consumergroup_group_offset{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"})",
 613           "format": "time_series",
 614           "hide": false,
 615           "intervalFactor": 1,
 616           "legendFormat": "Sum of group offsets",
 617           "refId": "B"
 618         },
 619         {
 620           "expr": "sum((kafka_consumergroup_group_offset{namespace=\"$namespace\",cluster_name=\"$cluster_name\",group=~\"$consumer_group\",group!=\"\"} * 0)\n+ on(namespace,cluster_name,topic,partition) group_left() kafka_partition_latest_offset{namespace=\"$namespace\",cluster_name=\"$cluster_name\"})",
 621           "format": "time_series",
 622           "intervalFactor": 1,
 623           "legendFormat": "Sum of latest offsets",
 624           "refId": "C"
 625         }
 626       ],
 627       "thresholds": [],
 628       "timeFrom": null,
 629       "timeShift": null,
 630       "title": "$consumer_group",
 631       "tooltip": {
 632         "shared": true,
 633         "sort": 0,
 634         "value_type": "individual"
 635       },
 636       "transparent": false,
 637       "type": "graph",
 638       "xaxis": {
 639         "buckets": null,
 640         "mode": "time",
 641         "name": null,
 642         "show": true,
 643         "values": []
 644       },
 645       "yaxes": [
 646         {
 647           "format": "s",
 648           "label": "estimated lag in seconds",
 649           "logBase": 1,
 650           "max": null,
 651           "min": "0",
 652           "show": true
 653         },
 654         {
 655           "format": "short",
 656           "label": "offset",
 657           "logBase": 1,
 658           "max": null,
 659           "min": null,
 660           "show": true
 661         }
 662       ],
 663       "yaxis": {
 664         "align": false,
 665         "alignLevel": null
 666       }
 667     },
 668     {
 669       "collapsed": false,
 670       "gridPos": {
 671         "h": 1,
 672         "w": 24,
 673         "x": 0,
 674         "y": 39
 675       },
 676       "id": 101,
 677       "panels": [],
 678       "title": "Kafka Lag Exporter JVM Metrics",
 679       "type": "row"
 680     },
 681     {
 682       "aliasColors": {},
 683       "bars": false,
 684       "dashLength": 10,
 685       "dashes": false,
 686       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 687       "fill": 1,
 688       "gridPos": {
 689         "h": 7,
 690         "w": 8,
 691         "x": 0,
 692         "y": 40
 693       },
 694       "id": 99,
 695       "legend": {
 696         "avg": false,
 697         "current": false,
 698         "max": false,
 699         "min": false,
 700         "show": true,
 701         "total": false,
 702         "values": false
 703       },
 704       "lines": true,
 705       "linewidth": 1,
 706       "links": [],
 707       "nullPointMode": "null",
 708       "percentage": false,
 709       "pointradius": 5,
 710       "points": false,
 711       "renderer": "flot",
 712       "seriesOverrides": [],
 713       "spaceLength": 10,
 714       "stack": false,
 715       "steppedLine": false,
 716       "targets": [
 717         {
 718           "expr": "sum(rate(jvm_memory_bytes_used{app_kubernetes_io_instance=\"kafka-lag-exporter\"}[5m])) by (kubernetes_pod_name)",
 719           "format": "time_series",
 720           "intervalFactor": 1,
 721           "legendFormat": "{{kubernetes_pod_name}}",
 722           "refId": "A"
 723         }
 724       ],
 725       "thresholds": [],
 726       "timeFrom": null,
 727       "timeShift": null,
 728       "title": "JVM Memory Used",
 729       "tooltip": {
 730         "shared": true,
 731         "sort": 0,
 732         "value_type": "individual"
 733       },
 734       "type": "graph",
 735       "xaxis": {
 736         "buckets": null,
 737         "mode": "time",
 738         "name": null,
 739         "show": true,
 740         "values": []
 741       },
 742       "yaxes": [
 743         {
 744           "format": "decbytes",
 745           "label": null,
 746           "logBase": 1,
 747           "max": null,
 748           "min": null,
 749           "show": true
 750         },
 751         {
 752           "format": "short",
 753           "label": null,
 754           "logBase": 1,
 755           "max": null,
 756           "min": null,
 757           "show": true
 758         }
 759       ],
 760       "yaxis": {
 761         "align": false,
 762         "alignLevel": null
 763       }
 764     },
 765     {
 766       "aliasColors": {},
 767       "bars": false,
 768       "dashLength": 10,
 769       "dashes": false,
 770       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 771       "fill": 1,
 772       "gridPos": {
 773         "h": 7,
 774         "w": 8,
 775         "x": 8,
 776         "y": 40
 777       },
 778       "id": 95,
 779       "legend": {
 780         "avg": false,
 781         "current": false,
 782         "max": false,
 783         "min": false,
 784         "show": true,
 785         "total": false,
 786         "values": false
 787       },
 788       "lines": true,
 789       "linewidth": 1,
 790       "links": [],
 791       "nullPointMode": "null",
 792       "percentage": false,
 793       "pointradius": 5,
 794       "points": false,
 795       "renderer": "flot",
 796       "seriesOverrides": [],
 797       "spaceLength": 10,
 798       "stack": false,
 799       "steppedLine": false,
 800       "targets": [
 801         {
 802           "expr": "sum(rate(jvm_gc_collection_seconds_sum{app_kubernetes_io_instance=\"kafka-lag-exporter\"}[5m])) by (kubernetes_pod_name)",
 803           "format": "time_series",
 804           "intervalFactor": 1,
 805           "legendFormat": "{{kubernetes_pod_name}}",
 806           "refId": "A"
 807         }
 808       ],
 809       "thresholds": [],
 810       "timeFrom": null,
 811       "timeShift": null,
 812       "title": "JVM GC Time",
 813       "tooltip": {
 814         "shared": true,
 815         "sort": 0,
 816         "value_type": "individual"
 817       },
 818       "type": "graph",
 819       "xaxis": {
 820         "buckets": null,
 821         "mode": "time",
 822         "name": null,
 823         "show": true,
 824         "values": []
 825       },
 826       "yaxes": [
 827         {
 828           "format": "s",
 829           "label": null,
 830           "logBase": 1,
 831           "max": null,
 832           "min": null,
 833           "show": true
 834         },
 835         {
 836           "format": "short",
 837           "label": null,
 838           "logBase": 1,
 839           "max": null,
 840           "min": null,
 841           "show": false
 842         }
 843       ],
 844       "yaxis": {
 845         "align": false,
 846         "alignLevel": null
 847       }
 848     },
 849     {
 850       "aliasColors": {},
 851       "bars": false,
 852       "dashLength": 10,
 853       "dashes": false,
 854       "datasource": "${DS_CINNAMON_PROMETHEUS}",
 855       "fill": 1,
 856       "gridPos": {
 857         "h": 7,
 858         "w": 8,
 859         "x": 16,
 860         "y": 40
 861       },
 862       "id": 97,
 863       "legend": {
 864         "avg": false,
 865         "current": false,
 866         "max": false,
 867         "min": false,
 868         "show": true,
 869         "total": false,
 870         "values": false
 871       },
 872       "lines": true,
 873       "linewidth": 1,
 874       "links": [],
 875       "nullPointMode": "null",
 876       "percentage": false,
 877       "pointradius": 5,
 878       "points": false,
 879       "renderer": "flot",
 880       "seriesOverrides": [],
 881       "spaceLength": 10,
 882       "stack": false,
 883       "steppedLine": false,
 884       "targets": [
 885         {
 886           "expr": "sum(rate(jvm_gc_collection_seconds_count{app_kubernetes_io_instance=\"kafka-lag-exporter\"}[5m])) by (kubernetes_pod_name)",
 887           "format": "time_series",
 888           "intervalFactor": 1,
 889           "legendFormat": "{{kubernetes_pod_name}}",
 890           "refId": "A"
 891         }
 892       ],
 893       "thresholds": [],
 894       "timeFrom": null,
 895       "timeShift": null,
 896       "title": "JVM GC Rate",
 897       "tooltip": {
 898         "shared": true,
 899         "sort": 0,
 900         "value_type": "individual"
 901       },
 902       "type": "graph",
 903       "xaxis": {
 904         "buckets": null,
 905         "mode": "time",
 906         "name": null,
 907         "show": true,
 908         "values": []
 909       },
 910       "yaxes": [
 911         {
 912           "format": "ops",
 913           "label": null,
 914           "logBase": 1,
 915           "max": null,
 916           "min": null,
 917           "show": true
 918         },
 919         {
 920           "format": "short",
 921           "label": null,
 922           "logBase": 1,
 923           "max": null,
 924           "min": null,
 925           "show": false
 926         }
 927       ],
 928       "yaxis": {
 929         "align": false,
 930         "alignLevel": null
 931       }
 932     }
 933   ],
 934   "refresh": "5m",
 935   "schemaVersion": 16,
 936   "style": "dark",
 937   "tags": [],
 938   "templating": {
 939     "list": [
 940       {
 941         "allValue": null,
 942         "current": {},
 943         "datasource": "${DS_CINNAMON_PROMETHEUS}",
 944         "hide": 0,
 945         "includeAll": false,
 946         "label": "Namespace",
 947         "multi": false,
 948         "name": "namespace",
 949         "options": [],
 950         "query": "query_result(kafka_consumergroup_group_lag)",
 951         "refresh": 1,
 952         "regex": "/.*namespace=\"([^\"]*).*/",
 953         "skipUrlSync": false,
 954         "sort": 1,
 955         "tagValuesQuery": "",
 956         "tags": [],
 957         "tagsQuery": "",
 958         "type": "query",
 959         "useTags": false
 960       },
 961       {
 962         "allValue": null,
 963         "current": {},
 964         "datasource": "${DS_CINNAMON_PROMETHEUS}",
 965         "hide": 0,
 966         "includeAll": false,
 967         "label": "Cluster Name",
 968         "multi": false,
 969         "name": "cluster_name",
 970         "options": [],
 971         "query": "query_result(kafka_consumergroup_group_lag{namespace=\"$namespace\"})",
 972         "refresh": 1,
 973         "regex": "/.*cluster_name=\"([^\"]*).*/",
 974         "skipUrlSync": false,
 975         "sort": 1,
 976         "tagValuesQuery": "",
 977         "tags": [],
 978         "tagsQuery": "",
 979         "type": "query",
 980         "useTags": false
 981       },
 982       {
 983         "allValue": ".*",
 984         "current": {},
 985         "datasource": "${DS_CINNAMON_PROMETHEUS}",
 986         "hide": 0,
 987         "includeAll": true,
 988         "label": "Consumer Group",
 989         "multi": true,
 990         "name": "consumer_group",
 991         "options": [],
 992         "query": "query_result(kafka_consumergroup_group_lag{namespace=\"$namespace\",cluster_name=\"$cluster_name\"})",
 993         "refresh": 1,
 994         "regex": "/.*group=\"([^\"]*).*/",
 995         "skipUrlSync": false,
 996         "sort": 1,
 997         "tagValuesQuery": "",
 998         "tags": [],
 999         "tagsQuery": "",
1000         "type": "query",
1001         "useTags": false
1002       }
1003     ]
1004   },
1005   "time": {
1006     "from": "now-30m",
1007     "to": "now"
1008   },
1009   "timepicker": {
1010     "refresh_intervals": [
1011       "5s",
1012       "10s",
1013       "30s",
1014       "1m",
1015       "5m",
1016       "15m",
1017       "30m",
1018       "1h",
1019       "2h",
1020       "1d"
1021     ],
1022     "time_options": [
1023       "5m",
1024       "15m",
1025       "1h",
1026       "6h",
1027       "12h",
1028       "24h",
1029       "2d",
1030       "7d",
1031       "30d"
1032     ]
1033   },
1034   "timezone": "",
1035   "title": "Kafka Lag Exporter",
1036   "uid": "8LW1Yd8ik",
1037   "version": 14
1038 }