AMBARI-18439 - [Grafana] Add Kafka-Offset dashboard for Storm (prajwal)

ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js

@@ -418,6 +418,21 @@ define([
               }));
             }
 
+            //Templatized Dashboard for Storm Kafka Offset
+            if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] &&
+                templateSrv.variables[1].name === "topic") {
+              var selectedTopology = templateSrv._values.topologies;
+              var selectedTopic = templateSrv._values.topic;
+              metricsPromises.push(_.map(options.targets, function(target) {
+                target.sTopology = selectedTopology;
+                target.sTopic = selectedTopic;
+                target.sPartition = options.scopedVars.partition.value;
+                target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic)
+                    .replace('*', target.sPartition);
+                return getStormData(target);
+              }));
+            }
+
             // To speed up querying on templatized dashboards.
             if (templateSrv.variables[1] && templateSrv.variables[1].name === "hosts") {
               var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? templateSrv._values.hosts.slice(1,-1) :
@@ -616,6 +631,55 @@ define([
                   });
                 });
           }
+          var stormEntities = {};
+          AmbariMetricsDatasource.prototype.getStormEntities = function () {
+            return this.initMetricAppidMapping()
+                .then(function () {
+                  var storm = allMetrics["nimbus"];
+                  var extractTopologies = storm.filter(/./.test.bind(new
+                      RegExp("partition", 'g')));
+                  _.map(extractTopologies, function(topology){
+                    topology = topology.split('.').slice(0,5);
+                    var topologyName = topologyN = topology[1]; // Topology
+                    var topologyTopicName = topicN = topology[3]; // Topic
+                    var topologyTopicPartitionName = topology[4]; // Partition
+                    if (stormEntities[topologyName]) {
+                      if (stormEntities[topologyName][topologyTopicName]) {
+                        stormEntities[topologyName][topologyTopicName].push(topologyTopicPartitionName);
+                      } else {
+                        stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName];
+                      }
+                    } else {
+                      stormEntities[topologyName] = {};
+                      stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName];
+                    }
+                  });
+                });
+          };
+          //Templated Variables for Storm Topics per Topology
+          if (interpolated.includes("stormTopic")) {
+            var topicName = interpolated.substring(0,interpolated.indexOf('.'));
+            return this.getStormEntities().then(function () {
+              var topicNames = Object.keys(stormEntities[topicName]);
+              return _.map(topicNames, function(names){
+                return {
+                  text: names
+                };
+              });
+            });
+          }
+          //Templated Variables for Storm Partitions per Topic
+          if (interpolated.includes("stormPartition")) {
+            var topicN, topologyN;
+            return this.getStormEntities().then(function () {
+              var partitionNames = _.uniq(stormEntities[topologyN][topicN]);
+              return _.map(partitionNames, function(names){
+                return {
+                  text: names
+                };
+              });
+            });
+          }
           // Templated Variable for YARN Queues.
           // It will search the cluster and populate the queues.
           if(interpolated === "yarnqueues") {

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json

@@ -0,0 +1,258 @@
+{
+  "id": null,
+  "title": "Storm - Kafka-Offset",
+  "originalTitle": "Storm - Kafka-Offset",
+  "tags": [
+    "storm",
+    "builtin",
+    "2.4.0.0"
+  ],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "25px",
+      "panels": [
+        {
+          "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari.  You may lose any changes made to this dashboard.  If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>",
+          "editable": true,
+          "error": false,
+          "id": 14,
+          "isNew": true,
+          "links": [],
+          "mode": "html",
+          "span": 12,
+          "style": {},
+          "title": "",
+          "type": "text"
+        }
+      ],
+      "title": "New row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 15,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "minSpan": 4,
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": "partition",
+          "scopedVars": {
+            "partition": {
+              "text": "partition-0",
+              "value": "partition-0",
+              "selected": true
+            }
+          },
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "alias": "Kafka Offset",
+              "app": "nimbus",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "",
+              "metric": "topology.*.kafka-topic.*.*.latestTimeOffset",
+              "precision": "default",
+              "refId": "B",
+              "sPartition": "partition-0",
+              "sTopic": "myKafkaTopic",
+              "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset",
+              "sTopology": "kafka-topology",
+              "seriesAggregator": "none",
+              "transform": "none"
+            },
+            {
+              "aggregator": "avg",
+              "alias": "Storm Offset",
+              "app": "nimbus",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset",
+              "precision": "default",
+              "refId": "A",
+              "sComponent": "--acker",
+              "sPartition": "partition-0",
+              "sTopic": "myKafkaTopic",
+              "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset",
+              "sTopology": "kafka-topology",
+              "seriesAggregator": "none",
+              "transform": "none"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "$partition",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    }
+  ],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "kafka-topology",
+          "value": "kafka-topology"
+        },
+        "datasource": null,
+        "includeAll": false,
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "topologies",
+        "options": [
+        ],
+        "query": "topologies",
+        "refresh": true,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "datasource": null,
+        "includeAll": false,
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "topic",
+        "options": [
+          
+        ],
+        "query": "$topologies.stormTopic",
+        "refresh": true,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "partition-0",
+          "value": "partition-0"
+        },
+        "datasource": null,
+        "includeAll": true,
+        "multi": true,
+        "multiFormat": "glob",
+        "name": "partition",
+        "options": [
+          {
+            "text": "partition-0",
+            "value": "partition-0",
+            "selected": true
+          }
+        ],
+        "query": "$topic.stormPartition",
+        "refresh": true,
+        "type": "query"
+      }
+    ]
+  },
+  "annotations": {
+    "list": []
+  },
+  "refresh": false,
+  "schemaVersion": 8,
+  "version": 23,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "tags": [
+        "storm"
+      ],
+      "title": "Storm Dashboards",
+      "type": "dashboards"
+    }
+  ]
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json

@@ -0,0 +1,258 @@
+{
+  "id": null,
+  "title": "Storm - Kafka-Offset",
+  "originalTitle": "Storm - Kafka-Offset",
+  "tags": [
+    "storm",
+    "builtin",
+    "2.4.0.0"
+  ],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "25px",
+      "panels": [
+        {
+          "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari.  You may lose any changes made to this dashboard.  If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>",
+          "editable": true,
+          "error": false,
+          "id": 14,
+          "isNew": true,
+          "links": [],
+          "mode": "html",
+          "span": 12,
+          "style": {},
+          "title": "",
+          "type": "text"
+        }
+      ],
+      "title": "New row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 15,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "minSpan": 4,
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": "partition",
+          "scopedVars": {
+            "partition": {
+              "text": "partition-0",
+              "value": "partition-0",
+              "selected": true
+            }
+          },
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "alias": "Kafka Offset",
+              "app": "nimbus",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "",
+              "metric": "topology.*.kafka-topic.*.*.latestTimeOffset",
+              "precision": "default",
+              "refId": "B",
+              "sPartition": "partition-0",
+              "sTopic": "myKafkaTopic",
+              "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset",
+              "sTopology": "kafka-topology",
+              "seriesAggregator": "none",
+              "transform": "none"
+            },
+            {
+              "aggregator": "avg",
+              "alias": "Storm Offset",
+              "app": "nimbus",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset",
+              "precision": "default",
+              "refId": "A",
+              "sComponent": "--acker",
+              "sPartition": "partition-0",
+              "sTopic": "myKafkaTopic",
+              "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset",
+              "sTopology": "kafka-topology",
+              "seriesAggregator": "none",
+              "transform": "none"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "$partition",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    }
+  ],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "kafka-topology",
+          "value": "kafka-topology"
+        },
+        "datasource": null,
+        "includeAll": false,
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "topologies",
+        "options": [
+        ],
+        "query": "topologies",
+        "refresh": true,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "datasource": null,
+        "includeAll": false,
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "topic",
+        "options": [
+          
+        ],
+        "query": "$topologies.stormTopic",
+        "refresh": true,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "allFormat": "glob",
+        "current": {
+          "text": "partition-0",
+          "value": "partition-0"
+        },
+        "datasource": null,
+        "includeAll": true,
+        "multi": true,
+        "multiFormat": "glob",
+        "name": "partition",
+        "options": [
+          {
+            "text": "partition-0",
+            "value": "partition-0",
+            "selected": true
+          }
+        ],
+        "query": "$topic.stormPartition",
+        "refresh": true,
+        "type": "query"
+      }
+    ]
+  },
+  "annotations": {
+    "list": []
+  },
+  "refresh": false,
+  "schemaVersion": 8,
+  "version": 23,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "tags": [
+        "storm"
+      ],
+      "title": "Storm Dashboards",
+      "type": "dashboards"
+    }
+  ]
+}