AMBARI-15214. Add ability to Create pre-built Grafana dashboards. (swagle)

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-master.json

@@ -0,0 +1,241 @@
+{
+  "id": null,
+  "title": "HBase Master",
+  "originalTitle": "HBase Master",
+  "tags": [ "builtin" ],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapUsedM",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapMaxM",
+              "refId": "C"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemNonHeapMaxM",
+              "refId": "D"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "JVM Heap",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "Row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 2,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillis",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillisParNew",
+              "refId": "C"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "JVM GC Times",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    }
+  ],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 5,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hbase-rs.json

@@ -0,0 +1,241 @@
+{
+  "id": 9,
+  "title": "HBase RegionServer",
+  "originalTitle": "HBase RegionServer",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapCommittedM",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapUsedM",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapMaxM",
+              "refId": "C"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemNonHeapMaxM",
+              "refId": "D"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "JVM Heap",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "Row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 2,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillis",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "hbase",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.GcTimeMillisParNew",
+              "refId": "C"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "JVM GC Times",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    }
+  ],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 5,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-dn.json

@@ -0,0 +1,199 @@
+{
+  "id": 12,
+  "title": "HDFS Datanode",
+  "originalTitle": "HDFS Datanode",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "rpc.rpc.RpcQueueTimeNumOps on sid-ams-1.novalocal",
+              "yaxis": 2
+            }
+          ],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "",
+              "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+              "refId": "A",
+              "templatedHost": "sid-ams-1.novalocal"
+            },
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "rpc.rpc.RpcQueueTimeNumOps",
+              "refId": "B",
+              "shouldComputeRate": true,
+              "templatedHost": "sid-ams-1.novalocal"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "RPC Average Queue Time",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "ms",
+            "short"
+          ]
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "format": "none",
+          "id": 2,
+          "interval": null,
+          "isNew": true,
+          "links": [],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "span": 6,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "sid-ams-1.novalocal",
+              "metric": "rpc.rpc.CallQueueLength",
+              "refId": "A"
+            }
+          ],
+          "thresholds": "",
+          "title": "RPC Call Queue Length",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        }
+      ],
+      "title": "Row"
+    }
+  ],
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 8,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-hdfs-nn.json

@@ -0,0 +1,199 @@
+{
+  "id": null,
+  "title": "HDFS Namenode",
+  "originalTitle": "HDFS Namenode",
+  "tags": [ "builtin" ],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "rpc.rpc.RpcQueueTimeNumOps on sid-ams-1.novalocal",
+              "yaxis": 2
+            }
+          ],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "",
+              "metric": "rpc.rpc.RpcQueueTimeAvgTime",
+              "refId": "A",
+              "templatedHost": "sid-ams-1.novalocal"
+            },
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "rpc.rpc.RpcQueueTimeNumOps",
+              "refId": "B",
+              "shouldComputeRate": true,
+              "templatedHost": "sid-ams-1.novalocal"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "RPC Average Queue Time",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "ms",
+            "short"
+          ]
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "format": "none",
+          "id": 2,
+          "interval": null,
+          "isNew": true,
+          "links": [],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "span": 6,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
+          },
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "namenode",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "sid-ams-1.novalocal",
+              "metric": "rpc.rpc.CallQueueLength",
+              "refId": "A"
+            }
+          ],
+          "thresholds": "",
+          "title": "RPC Call Queue Length",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        }
+      ],
+      "title": "Row"
+    }
+  ],
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 8,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-apps.json

@@ -0,0 +1,135 @@
+{
+  "id": 5,
+  "title": "YARN Applications",
+  "originalTitle": "YARN Applications",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "AMS",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.AMLaunchDelayAvgTime",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.AMRegisterDelayAvgTime",
+              "refId": "B"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "AM launch / register delay",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "µs",
+            "short"
+          ]
+        }
+      ],
+      "title": "Row"
+    }
+  ],
+  "time": {
+    "from": "now-30d",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 2,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-nm.json

@@ -0,0 +1,315 @@
+{
+  "id": 4,
+  "title": "Node Managers",
+  "originalTitle": "Node Managers",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "content": "This dashboard is managed by Ambari.\nIf you make any changes to this, Ambari may refresh this dashboard anytime and you could lose your change.\nIf you want to customize this dashboard, please make a copy first.",
+          "editable": true,
+          "error": false,
+          "id": 4,
+          "isNew": true,
+          "links": [],
+          "mode": "markdown",
+          "span": 6,
+          "style": {},
+          "title": "Node Managers Dashboard",
+          "type": "text"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "AMS",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "hosts": "",
+              "metric": "yarn.NodeManagerMetrics.AllocatedGB",
+              "refId": "A",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            },
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.NodeManagerMetrics.AllocatedVCores",
+              "refId": "B",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Memory / Vcore",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "AMS",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 3,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc",
+              "refId": "A",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            },
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc",
+              "refId": "B",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Disk Utilization Percentage",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "AMS",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 2,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.NodeManagerMetrics.BadLocalDirs",
+              "refId": "A",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            },
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.NodeManagerMetrics.BadLogDirs",
+              "refId": "B",
+              "templatedHost": "perf-a-2.c.pramod-thangali.internal"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Bad local / log dir",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "Row"
+    }
+  ],
+  "time": {
+    "from": "now-30d",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 11,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-overview.json

@@ -0,0 +1,488 @@
+{
+  "id": 3,
+  "title": "YARN Home",
+  "originalTitle": "YARN Home",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "200px",
+      "panels": [
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "format": "none",
+          "id": 3,
+          "interval": null,
+          "isNew": true,
+          "links": [],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "span": 4,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.NumActiveNMs",
+              "refId": "A"
+            }
+          ],
+          "thresholds": "",
+          "title": "Active nodes",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "format": "none",
+          "id": 2,
+          "interval": null,
+          "isNew": true,
+          "links": [],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "span": 4,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.QueueMetrics.Queue=root.AppsRunning",
+              "precision": "seconds",
+              "refId": "A",
+              "shouldAddPrecision": false
+            }
+          ],
+          "thresholds": "",
+          "title": "Running apps",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        },
+        {
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "format": "mbytes",
+          "id": 6,
+          "interval": null,
+          "isNew": true,
+          "links": [],
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "span": 4,
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": true
+          },
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "jvm.JvmMetrics.MemHeapUsedM",
+              "refId": "C"
+            }
+          ],
+          "thresholds": "",
+          "title": "ResourceManager Heap",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
+        }
+      ],
+      "title": "Row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 8,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [
+            {
+              "dashUri": "db/node-managers",
+              "dashboard": "Node Managers",
+              "title": "Node Managers",
+              "type": "dashboard"
+            }
+          ],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.NumActiveNMs",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.NumUnhealthyNMs",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.ClusterMetrics.NumLostNMs",
+              "refId": "C"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Nodes",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 4,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [
+            {
+              "dashUri": "db/yarn-applications",
+              "dashboard": "YARN Applications",
+              "title": "YARN Applications",
+              "type": "dashboard"
+            }
+          ],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 6,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.QueueMetrics.Queue=root.AppsRunning",
+              "refId": "A"
+            },
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.QueueMetrics.Queue=root.AppsPending",
+              "refId": "B"
+            },
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.QueueMetrics.Queue=root.AppsFailed",
+              "refId": "C"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Apps",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "transparent": false,
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 9,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "resourcemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "yarn.QueueMetrics.Queue=root.AllocatedMB",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Cluster utilization",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "mbytes",
+            "short"
+          ]
+        }
+      ],
+      "title": "New row"
+    }
+  ],
+  "time": {
+    "from": "now-3d",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 22,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/grafana-yarn-rm.json

@@ -0,0 +1,125 @@
+{
+  "id": 6,
+  "title": "YARN ResourceManager",
+  "originalTitle": "YARN ResourceManager",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": false,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "hcube",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "leftLogBase": 1,
+            "leftMax": null,
+            "leftMin": null,
+            "rightLogBase": 1,
+            "rightMax": null,
+            "rightMin": null,
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "aggregator": "avg",
+              "app": "nodemanager",
+              "downsampleAggregator": "avg",
+              "errors": {},
+              "metric": "",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Panel Title",
+          "tooltip": {
+            "shared": true,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "x-axis": true,
+          "y-axis": true,
+          "y_formats": [
+            "short",
+            "short"
+          ]
+        }
+      ],
+      "title": "Row"
+    }
+  ],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "schemaVersion": 8,
+  "version": 1,
+  "links": []
+}

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py

@@ -22,7 +22,7 @@ from resource_management import Script, Execute
 from resource_management.libraries.functions import format
 from status import check_service_status
 from ams import ams
-from metrics_grafana_util import create_ams_datasource
+from metrics_grafana_util import create_ams_datasource, create_ams_dashboards
 
 class AmsGrafana(Script):
   def install(self, env):
@@ -50,7 +50,10 @@ class AmsGrafana(Script):
     Execute(start_cmd,
             user=params.ams_user
             )
+    # Create datasource
     create_ams_datasource()
+    # Create pre-built dashboards
+    create_ams_dashboards()
 
   def stop(self, env):
     import params

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py

@@ -21,135 +21,271 @@ import httplib
 from resource_management.core.logger import Logger
 from resource_management.core.base import Fail
 from resource_management import Template
-
+from collections import namedtuple
+from urlparse import urlparse
 import time
 import socket
-import json
+import ambari_simplejson as json
 import network
 
-def create_ams_datasource():
+GRAFANA_CONNECT_TRIES = 5
+GRAFANA_CONNECT_TIMEOUT = 15
+GRAFANA_SEARCH_BULTIN_DASHBOARDS = "/api/search?tag=builtin"
+GRAFANA_DATASOURCE_URL = "/api/datasources"
+GRAFANA_DASHBOARDS_URL = "/api/dashboards/db"
+METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS"
+
+Server = namedtuple('Server', [ 'protocol', 'host', 'port' ])
+
+def perform_grafana_get_call(url, server):
+  grafana_https_enabled = server.protocol.lower() == 'https'
+
+  conn = network.get_http_connection(server.host,
+                                     int(server.port),
+                                     grafana_https_enabled)
+
+  Logger.info("Connecting (GET) to %s:%s%s" % (server.host,
+                                               server.port,
+                                               url))
+  conn.request("GET", url)
+  response = conn.getresponse()
+  Logger.info("Http response: %s %s" % (response.status, response.reason))
+
+  return response
+
+def perform_grafana_put_call(url, id, payload, server):
+  response = None
+  data = None
+  headers = {"Content-Type": "application/json"}
+  grafana_https_enabled = server.protocol.lower() == 'https'
+
+  for i in xrange(0, GRAFANA_CONNECT_TRIES):
+    try:
+      conn = network.get_http_connection(server.host, int(server.port), grafana_https_enabled)
+      conn.request("PUT", url + "/" + str(id), payload, headers)
+      response = conn.getresponse()
+      data = response.read()
+      Logger.info("Http data: %s" % data)
+      conn.close()
+      break
+    except (httplib.HTTPException, socket.error) as ex:
+      if i < GRAFANA_CONNECT_TRIES - 1:
+        time.sleep(GRAFANA_CONNECT_TIMEOUT)
+        Logger.info("Connection to Grafana failed. Next retry in %s seconds."
+                    % (GRAFANA_CONNECT_TIMEOUT))
+        continue
+      else:
+        raise Fail("Ambari Metrics Grafana update failed")
+
+  return (response, data)
+
+def perform_grafana_post_call(url, payload, server):
+  response = None
+  data = None
+  Logger.debug('POST payload: %s' % payload)
+  headers = {"Content-Type": "application/json", "Content-Length" : len(payload)}
+  grafana_https_enabled = server.protocol.lower() == 'https'
 
+  for i in xrange(0, GRAFANA_CONNECT_TRIES):
+    try:
+      Logger.info("Connecting (POST) to %s:%s%s" % (server.host, server.port, url))
+      conn = network.get_http_connection(server.host,
+                                         int(server.port),
+                                         grafana_https_enabled)
+      
+      conn.request("POST", url, payload, headers)
+
+      response = conn.getresponse()
+      Logger.info("Http response: %s %s" % (response.status, response.reason))
+      data = response.read()
+      Logger.info("Http data: %s" % data)
+      conn.close()
+      break
+    except (httplib.HTTPException, socket.error) as ex:
+      if i < GRAFANA_CONNECT_TRIES - 1:
+        time.sleep(GRAFANA_CONNECT_TIMEOUT)
+        Logger.info("Connection to Grafana failed. Next retry in %s seconds."
+                    % (GRAFANA_CONNECT_TIMEOUT))
+        continue
+      else:
+        raise Fail("Ambari Metrics Grafana create failed.")
+
+  return (response, data)
+
+def is_unchanged_datasource_url(datasource_url):
+  import params
+  parsed_url = urlparse(datasource_url)
+  Logger.debug("parsed url, scheme = %s, host = %s, port = %s" % (
+    parsed_url.scheme, parsed_url.hostname, parsed_url.port))
+  Logger.debug("collector, scheme = %s, host = %s, port = %s" %
+              (params.metric_collector_protocol, params.metric_collector_host,
+               params.metric_collector_port))
+
+  return (parsed_url.scheme == params.metric_collector_protocol.strip()) and \
+         (parsed_url.hostname == params.metric_collector_host.strip()) and \
+         (parsed_url.port == params.metric_collector_port)
+
+def create_ams_datasource():
   import params
-  GRAFANA_CONNECT_TRIES = 5
-  GRAFANA_CONNECT_TIMEOUT = 15
-  GRAFANA_URL = "/api/datasources"
-  METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS"
-  grafana_https_enabled = params.ams_grafana_protocol.lower() == 'https'
-  headers = {"Content-type": "application/json"}
+  server = Server(protocol = params.ams_grafana_protocol.strip(),
+                  host = params.ams_grafana_host.strip(),
+                  port = params.ams_grafana_port)
 
+  """
+  Create AMS datasource in Grafana, if exsists make sure the collector url is accurate
+  """
   ams_datasource_json = Template('metrics_grafana_datasource.json.j2',
-                                 ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME)\
-                                 .get_content()
+                                 ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME).get_content()
 
   Logger.info("Checking if AMS Grafana datasource already exists")
-  Logger.info("Connecting (GET) to %s:%s%s" % (params.hostname,
-                                               params.ams_grafana_port,
-                                               GRAFANA_URL))
 
-  conn = network.get_http_connection(params.hostname,
-                                     int(params.ams_grafana_port),
-                                     grafana_https_enabled)
 
-  conn.request("GET", GRAFANA_URL)
-  response = conn.getresponse()
-  Logger.info("Http response: %s %s" % (response.status, response.reason))
+  response = perform_grafana_get_call(GRAFANA_DATASOURCE_URL, server)
+  create_datasource = True
 
-  if(response.status == 200):
+  if response.status == 200:
     datasources = response.read()
     datasources_json = json.loads(datasources)
     for i in xrange(0, len(datasources_json)):
       datasource_name = datasources_json[i]["name"]
-      if(datasource_name == METRICS_GRAFANA_DATASOURCE_NAME):
-
+      if datasource_name == METRICS_GRAFANA_DATASOURCE_NAME:
+        create_datasource = False # datasource already exists
         Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL")
         datasource_url = datasources_json[i]["url"]
 
-        if datasource_url == (params.metric_collector_protocol + "://"
-                                + params.metric_collector_host + ":"
-                                + params.metric_collector_port):
-          Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation")
-          GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again
-
+        if is_unchanged_datasource_url(datasource_url):
+          Logger.info("Metrics Collector URL validation succeeded.")
+          return
         else: # Metrics datasource present, but collector host is wrong.
-
-          Logger.info("Metrics Collector URL validation failed.")
           datasource_id = datasources_json[i]["id"]
-          Logger.info("Deleting obselete Metrics datasource.")
-          conn = network.get_http_connection(params.hostname,
-                                             int(params.ams_grafana_port),
-                                             grafana_https_enabled)
-          conn.request("PUT", GRAFANA_URL + "/" + str(datasource_id), ams_datasource_json, headers)
-          response = conn.getresponse()
-          data = response.read()
-          Logger.info("Http data: %s" % data)
-          conn.close()
+          Logger.info("Metrics Collector URL validation failed. Updating "
+                      "datasource, id = %s" % datasource_id)
+
+          (response, data) = perform_grafana_put_call(GRAFANA_DATASOURCE_URL, datasource_id,
+                                                      ams_datasource_json, server)
 
           if response.status == 200:
             Logger.info("Ambari Metrics Grafana data source updated.")
-            GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again
+
           elif response.status == 500:
             Logger.info("Ambari Metrics Grafana data source update failed. Not retrying.")
             raise Fail("Ambari Metrics Grafana data source update failed. PUT request status: %s %s \n%s" %
                        (response.status, response.reason, data))
           else:
-            Logger.info("Ambari Metrics Grafana data source update failed.")
-            if i < GRAFANA_CONNECT_TRIES - 1:
-              time.sleep(GRAFANA_CONNECT_TIMEOUT)
-              Logger.info("Next retry in %s seconds."
-                          % (GRAFANA_CONNECT_TIMEOUT))
-            else:
-              raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
-                         (response.status, response.reason, data))
-
-        break
+            raise Fail("Ambari Metrics Grafana data source creation failed. "
+                       "PUT request status: %s %s \n%s" % (response.status, response.reason, data))
+        pass
+      pass
+    pass
   else:
     Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.")
 
-  if GRAFANA_CONNECT_TRIES > 0:
-    Logger.info("Attempting to create Ambari Metrics Grafana datasource")
-
-  for i in xrange(0, GRAFANA_CONNECT_TRIES):
-    try:
-
-      Logger.info("Generated datasource:\n%s" % ams_datasource_json)
-
-      Logger.info("Connecting (POST) to %s:%s%s" % (params.hostname,
-                                                    params.ams_grafana_port,
-                                                    GRAFANA_URL))
-      conn = network.get_http_connection(params.hostname,
-                                         int(params.ams_grafana_port),
-                                         grafana_https_enabled)
-      conn.request("POST", GRAFANA_URL, ams_datasource_json, headers)
-
-      response = conn.getresponse()
-      Logger.info("Http response: %s %s" % (response.status, response.reason))
-    except (httplib.HTTPException, socket.error) as ex:
-      if i < GRAFANA_CONNECT_TRIES - 1:
-        time.sleep(GRAFANA_CONNECT_TIMEOUT)
-        Logger.info("Connection to Grafana failed. Next retry in %s seconds."
-                    % (GRAFANA_CONNECT_TIMEOUT))
-        continue
-      else:
-        raise Fail("Ambari Metrics Grafana datasource not created")
+  if not create_datasource:
+    return
+  else:
+    Logger.info("Generating datasource:\n%s" % ams_datasource_json)
 
-    data = response.read()
-    Logger.info("Http data: %s" % data)
-    conn.close()
+    (response, data) = perform_grafana_post_call(GRAFANA_DATASOURCE_URL, ams_datasource_json, server)
 
     if response.status == 200:
       Logger.info("Ambari Metrics Grafana data source created.")
-      break
     elif response.status == 500:
       Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.")
       raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
                  (response.status, response.reason, data))
     else:
       Logger.info("Ambari Metrics Grafana data source creation failed.")
-      if i < GRAFANA_CONNECT_TRIES - 1:
-        time.sleep(GRAFANA_CONNECT_TIMEOUT)
-        Logger.info("Next retry in %s seconds."
-                  % (GRAFANA_CONNECT_TIMEOUT))
-      else:
-        raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
+      raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" %
                  (response.status, response.reason, data))
   pass
+
+def create_ams_dashboards():
+  """
+  Create dashboards in grafana from the json files
+  """
+  import params
+  server = Server(protocol = params.ams_grafana_protocol.strip(),
+                  host = params.ams_grafana_host.strip(),
+                  port = params.ams_grafana_port)
+
+  dashboard_files = params.get_grafana_dashboard_defs()
+  version = params.get_ambari_version()
+  Logger.info("Checking dashboards to update for Ambari version : %s" % version)
+  # Friendly representation of dashboard
+  Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags'])
+
+  existing_dashboards = []
+  response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS, server)
+  if response and response.status == 200:
+    data = response.read()
+    try:
+      datasources = json.loads(data)
+    except:
+      Logger.error("Unable to parse JSON response from grafana request: %s" %
+                   GRAFANA_SEARCH_BULTIN_DASHBOARDS)
+      Logger.info(data)
+      return
+
+    for datasource in datasources:
+      existing_dashboards.append(
+          Dashboard(uri = datasource['uri'], id = datasource['id'],
+                    title = datasource['title'], tags = datasource['tags'])
+        )
+    pass
+  else:
+    Logger.error("Failed to execute search query on Grafana dashboards. "
+                 "query = %s\n statuscode = %s\n reason = %s\n data = %s\n" %
+                 (GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status, response.reason, response.read()))
+    return
+
+  Logger.debug('Dashboard definitions found = %s' % str(dashboard_files))
+
+  if dashboard_files:
+    for dashboard_file in dashboard_files:
+      try:
+        with open(dashboard_file, 'r') as file:
+          dashboard_def = json.load(file)
+      except Exception, e:
+        Logger.error('Unable to load dashboard json file %s' % dashboard_file)
+        Logger.error(str(e))
+        continue
+
+      if dashboard_def:
+        update_def = True
+        # Make sure static json does not have id
+        if "id" in dashboard_def:
+          dashboard_def['id'] = None
+        # Set correct tags
+        dashboard_def['tags'] = [ 'builtin', version ]
+        dashboard_def['overwrite'] = True
+        
+        for dashboard in existing_dashboards:
+          if dashboard.title == dashboard_def['title']:
+            if version not in dashboard.tags:
+              # Found existing dashboard with wrong version - update dashboard
+              update_def = True
+            else:
+              update_def = False # Skip update
+        pass
+
+        if update_def:
+          Logger.info("Updating dashboard definition for %s with tags: %s" %
+                      (dashboard_def['title'], dashboard_def['tags']))
+
+          # Discrepancy in grafana export vs import format
+          dashboard_def_payload = { "dashboard" : dashboard_def }
+          paylaod = json.dumps(dashboard_def_payload).strip()
+
+          (response, data) = perform_grafana_post_call(GRAFANA_DASHBOARDS_URL, paylaod, server)
+
+          if response and response.status == 200:
+            Logger.info("Dashboard created successfully.\n %s" % str(data))
+          else:
+            Logger.error("Failed creating dashboard: %s" % dashboard_def['title'])
+          pass
+        pass
+      pass
+    pass
+
+
+

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -21,11 +21,12 @@ limitations under the License.
 from functions import calc_xmn_from_xms
 from functions import check_append_heap_property
 from functions import trim_heap_property
-
+from resource_management.core.logger import Logger
 from resource_management import *
 import status_params
 from ambari_commons import OSCheck
-
+import ConfigParser
+import os
 
 if OSCheck.is_windows_family():
   from params_windows import *
@@ -61,6 +62,41 @@ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.trust
 metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
 metric_truststore_ca_certs='ca.pem'
 
+agent_cache_dir = config['hostLevelParams']['agentCacheDir']
+service_package_folder = config['commandParams']['service_package_folder']
+dashboards_dir = os.path.join(agent_cache_dir, service_package_folder, 'files', 'grafana-dashboards')
+
+def get_grafana_dashboard_defs():
+  dashboard_defs = []
+  if os.path.exists(dashboards_dir):
+    for root, dirs, files in os.walk(dashboards_dir):
+      for file in files:
+        if 'grafana' in file:
+          dashboard_defs.append(os.path.join(root, file))
+  return dashboard_defs
+
+# find ambari version for grafana dashboards
+def get_ambari_version():
+  ambari_version = None
+  AMBARI_AGENT_CONF = '/etc/ambari-agent/conf/ambari-agent.ini'
+  ambari_agent_config = ConfigParser.RawConfigParser()
+  if os.path.exists(AMBARI_AGENT_CONF):
+    try:
+      ambari_agent_config.read(AMBARI_AGENT_CONF)
+      data_dir = ambari_agent_config.get('agent', 'prefix')
+      ver_file = os.path.join(data_dir, 'version')
+      f = open(ver_file, "r")
+      ambari_version = f.read().strip()
+      f.close()
+    except Exception, e:
+      Logger.info('Unable to determine ambari version from version file.')
+      Logger.debug('Exception: %s' % str(e))
+      # No hostname script identified in the ambari agent conf
+      pass
+    pass
+  return ambari_version
+
+
 if 'cluster-env' in config['configurations'] and \
     'metrics_collector_vip_host' in config['configurations']['cluster-env']:
   metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
@@ -92,6 +128,10 @@ ams_grafana_pid_dir = status_params.ams_grafana_pid_dir
 ams_grafana_conf_dir = '/etc/ambari-metrics-grafana/conf'
 ams_grafana_data_dir = default("/configurations/ams-grafana-env/metrics_grafana_data_dir", '/var/lib/ambari-metrics-grafana')
 
+metrics_grafana_hosts = default('/clusterHostInfo/metrics_grafana_hosts', None)
+ams_grafana_host = None
+if metrics_grafana_hosts:
+  ams_grafana_host = metrics_grafana_hosts[0]
 ams_grafana_port = default("/configurations/ams-grafana-ini/port", 3000)
 ams_grafana_protocol = default("/configurations/ams-grafana-ini/protocol", 'http')
 ams_grafana_cert_file = default("/configurations/ams-grafana-ini/cert_file", '/etc/ambari-metrics/conf/ams-grafana.crt')

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -3,6 +3,7 @@
     "clusterName": "c1",
     "hostname": "c6401.ambari.apache.org",
     "hostLevelParams": {
+        "agentCacheDir": "/var/lib/ambari-agent/cache",
         "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
         "ambari_db_rca_password": "mapred",
         "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
@@ -1138,6 +1139,9 @@
         ],
         "metrics_collector_hosts": [
             "c6401.ambari.apache.org"
+        ],
+        "metrics_grafana_hosts": [
+            "c6401.ambari.apache.org"
         ]
     }
 }

ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json

@@ -3,6 +3,7 @@
     "clusterName": "c1", 
     "hostname": "c6401.ambari.apache.org", 
     "hostLevelParams": {
+        "agentCacheDir": "/var/lib/ambari-agent/cache",
         "jdk_location": "http://c6401.ambari.apache.org:8080/resources/", 
         "ambari_db_rca_password": "mapred", 
         "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",