AMBARI-11195. Need to be able to create customized graphs based on YARN queue-specific metrics. (jaimin)

ambari-web/app/assets/test/tests.js

@@ -87,6 +87,7 @@ var files = ['test/init_model_test',
   'test/controllers/main/service/reassign/step4_controller_test',
   'test/controllers/main/service/reassign/step6_controller_test',
   'test/controllers/main/service/reassign/step7_controller_test',
+  'test/controllers/main/service/widgets/create/wizard_controller_test',
   'test/controllers/main/service/widgets/create/step1_controller_test',
   'test/controllers/main/service/widgets/create/step2_controller_test',
   'test/controllers/main/dashboard_test',

ambari-web/app/controllers/main/service/widgets/create/step2_controller.js

@@ -343,14 +343,17 @@ App.WidgetWizardStep2Controller = Em.Controller.extend({
       value = '${';
       expression.data.forEach(function (element) {
         if (element.isMetric) {
-          metrics.push({
+          var metricObj = {
             "name": element.name,
             "service_name": element.serviceName,
             "component_name": element.componentName,
-            "metric_path": element.metricPath,
-            "host_component_criteria": element.hostComponentCriteria,
-            "category": element.category
-          });
+            "metric_path": element.metricPath
+          };
+          if (element.hostComponentCriteria) {
+            metricObj.host_component_criteria = element.hostComponentCriteria;
+          }
+          metrics.push(metricObj);
+
         }
         value += element.name;
       }, this);

ambari-web/app/controllers/main/service/widgets/create/step3_controller.js

@@ -122,15 +122,7 @@ App.WidgetWizardStep3Controller = Em.Controller.extend({
         description: this.get('widgetDescription') || "",
         scope: this.get('widgetScope').toUpperCase(),
         author: this.get('widgetAuthor'),
-        "metrics": this.get('widgetMetrics').map(function (metric) {
-          return {
-            "name": metric.name,
-            "service_name": metric.serviceName,
-            "component_name": metric.componentName,
-            "metric_path": metric.metricPath,
-            "host_component_criteria": metric.hostComponentCriteria
-          }
-        }),
+        metrics: this.get('widgetMetrics'),
         values: this.get('widgetValues').map(function (value) {
           delete value.computedValue;
           return value;

ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js

@@ -180,17 +180,17 @@ App.WidgetWizardController = App.WizardController.extend({
    * @returns {$.Deferred}
    */
   loadAllMetrics: function () {
-    var widgetMetrics = this.getDBProperty('allMetrics');
+    var allMetrics = this.getDBProperty('allMetrics');
     var self = this;
     var dfd = $.Deferred();
 
-    if (widgetMetrics.length === 0) {
+    if (allMetrics.length === 0) {
       this.loadAllMetricsFromServer(function () {
         dfd.resolve(self.get('content.allMetrics'));
       });
     } else {
-      this.set('content.allMetrics', widgetMetrics);
-      dfd.resolve(widgetMetrics);
+      this.set('content.allMetrics', allMetrics);
+      dfd.resolve(allMetrics);
     }
     return dfd.promise();
   },
@@ -223,23 +223,25 @@ App.WidgetWizardController = App.WizardController.extend({
     var self = this;
     var result = [];
     var metrics = {};
-
+    var slaveComponents = App.StackServiceComponent.find().filterProperty('isSlave').mapProperty('componentName');
     if (json) {
       json.items.forEach(function (service) {
         var data = service.artifacts[0].artifact_data[service.StackServices.service_name];
         for (var componentName in data) {
+          var isSlave = slaveComponents.contains(componentName);
           for (var level in data[componentName]) {
             var metricTypes = data[componentName][level]; //Ganglia or JMX
             metricTypes.forEach(function (_metricType) {
               metrics = _metricType['metrics']['default'];
               var type = _metricType["type"].toUpperCase();
-              if (!(type === 'JMX' && level.toUpperCase() === 'COMPONENT')) {
+              if (!((type === 'JMX' && level.toUpperCase() === 'COMPONENT') || (isSlave && level.toUpperCase() === 'HOSTCOMPONENT'))) {
                 for (var widgetId in metrics) {
+                  var _metrics = metrics[widgetId];
                   var metricObj = {
                     widget_id: widgetId,
-                    point_in_time: metrics[widgetId].pointInTime,
-                    temporal: metrics[widgetId].temporal,
-                    name: metrics[widgetId].name,
+                    point_in_time: _metrics.pointInTime,
+                    temporal: _metrics.temporal,
+                    name: _metrics.name,
                     level: level.toUpperCase(),
                     type: type,
                     component_name: componentName,
@@ -256,9 +258,56 @@ App.WidgetWizardController = App.WizardController.extend({
         }
       }, this);
     }
+    if (!!App.YARNService.find("YARN")) {
+      result = this.substitueQueueMetrics(result);
+    }
     this.save('allMetrics', result);
   },
 
+
+  /**
+   * @name substitueQueueMetrics
+   * @param metrics
+   * @return {Array} array of metric objects with regex substituted with actual metrics names
+   */
+  substitueQueueMetrics: function (metrics) {
+    var result = [];
+    var queuePaths = App.YARNService.find("YARN").get("allQueueNames");
+    var queueNames = [];
+    var queueMetricName;
+    queuePaths.forEach(function (_queuePath) {
+      var queueName = _queuePath.replace(/\//g, ".");
+      queueNames.push(queueName);
+    }, this);
+    var regexpForAMS = new RegExp("^yarn.QueueMetrics.Queue=\\(\\.\\+\\).*$");
+    var regexpForJMX = new RegExp("\\(\\.\\+\\)", "g");
+    var replaceRegexForMetricName = regexpForJMX;
+    var replaceRegexForMetricPath = new RegExp("\\$\\d\\..*\\)(?=\\/)", "g");
+    metrics.forEach(function (_metric) {
+      var isAMSQueueMetric = regexpForAMS.test(_metric.name);
+      var isJMXQueueMetrics = regexpForJMX.test(_metric.name);
+      if ((_metric.type === 'GANGLIA' && isAMSQueueMetric) || (_metric.type === 'JMX' && isJMXQueueMetrics)) {
+        queuePaths.forEach(function (_queuePath) {
+          queueMetricName = '';
+          if (_metric.type === 'GANGLIA') {
+            queueMetricName = _queuePath.replace(/\//g, ".");
+          } else if (_metric.type === 'JMX') {
+            _queuePath.split("/").forEach(function(_metricName, index){
+              queueMetricName = queueMetricName + ',q' + index + '=' + _metricName;
+            }, this);
+          }
+          var metricName = _metric.name.replace(replaceRegexForMetricName, queueMetricName);
+          var newMetricPath = _metric.widget_id.replace(replaceRegexForMetricPath, _queuePath);
+          var newQueueMetric = $.extend(true, {}, _metric, {name: metricName, widget_id: newMetricPath});
+          result.pushObject(newQueueMetric);
+        }, this);
+      } else {
+        result.pushObject(_metric);
+      }
+    }, this);
+    return result;
+  },
+
   /**
    *
    * @param metricObj {Object}

ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js

@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App = require('app');
+
+require('controllers/main/service/widgets/create/wizard_controller');
+
+
+describe('App.WidgetWizardController', function () {
+  var controller;
+
+  /**
+   * tests the function with following hierarchical queue scenario
+   *                   root 
+   *                    |
+   *                  queue1
+   *                 /     \
+   *             queue2   queue3
+   *                  
+   */
+  describe("#substitueQueueMetrics", function () {
+    beforeEach(function () {
+      controller = App.WidgetWizardController.create();
+      sinon.stub(App.YARNService, 'find', function (k) {
+        if ('YARN' === k) return Em.Object.create({
+          'allQueueNames': ["root", "root/queue1", "root/queue1/queue2", "root/queue1/queue3"]
+        });
+      });
+    });
+    afterEach(function () {
+      controller = '';
+      App.YARNService.find.restore();
+    });
+
+
+    var testCases = [
+      {
+        msg: 'AMS Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue=(.+).AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/$1.replaceAll("([.])","/")/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.queue2.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/queue2/AppsFailed"
+          },
+          {
+            "component_name": "RESOURCEMANAGER",
+            "level": "COMPONENT",
+            "name": "yarn.QueueMetrics.Queue=root.queue1.queue3.AppsFailed",
+            "point_in_time": false,
+            "service_name": "YARN",
+            "temporal": true,
+            "type": "GANGLIA",
+            "widget_id": "metrics/yarn/Queue/root/queue1/queue3/AppsFailed"
+          }
+        ]
+      },
+      {
+        msg: 'JMX Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/$1.replaceAll(",q(\d+)=","/").substring(1)/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue2.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/queue2/AppsFailed'
+          },
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue3.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/root/queue1/queue3/AppsFailed'
+          }
+        ]
+      },
+      {
+        msg: 'AMS Queue metric without regex in name and path should retain same name and path',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            level: 'COMPONENT',
+            name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+            point_in_time: false,
+            service_name: 'YARN',
+            temporal: true,
+            type: 'GANGLIA',
+            widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+          }
+        ]
+      },
+      {
+        msg: 'JMX Queue metric without regex in name and path should retain same name and path',
+        inputMetrics: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+          }
+        ],
+        expectedResult: [
+          {
+            component_name: 'RESOURCEMANAGER',
+            host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+            level: 'HOSTCOMPONENT',
+            name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+            point_in_time: true,
+            service_name: 'YARN',
+            temporal: false,
+            type: 'JMX',
+            widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+          }
+        ]
+      }
+    ];
+    testCases.forEach(function (_testCase) {
+      it(_testCase.msg, function () {
+        var result = controller.substitueQueueMetrics(_testCase.inputMetrics);
+        expect(JSON.stringify(result)).to.equal(JSON.stringify(_testCase.expectedResult));
+      });
+    });
+  });
+});