AMBARI-4922 Storm: Add service graphs. (Denys Buzhor via atkach)

ambari-web/app/assets/data/services/metrics/storm/storm_rest_api_metrics.json

@@ -0,0 +1,19 @@
+{
+  "href" : "http://ambari/clusters/c1/services/STORM/component/STORM_REST_API?fields=metrics",
+  "HostRoles" : {
+    "cluster_name" : "c1",
+    "component_name" : "STORM_REST_API",
+    "service_name" : "STORM"
+  },
+  "host" : {
+    "href" : "http://ambari/clusters/c1/hosts/hostname"
+  },
+  "metrics" : {
+    "topologies": 1,
+    "executors.total": 5,
+    "slots.total": 4,
+    "slots.used": 1,
+    "tasks.total": 2,
+    "slots.free": 3
+  }
+}

ambari-web/app/controllers/global/update_controller.js

@@ -134,7 +134,8 @@ App.UpdateController = Em.Controller.extend({
       'MAPREDUCE': "ServiceComponentInfo/AliveNodes," +
                    "ServiceComponentInfo/GrayListedNodes," +
                    "ServiceComponentInfo/BlackListedNodes," +
-                   "ServiceComponentInfo/jobtracker/*,"
+                   "ServiceComponentInfo/jobtracker/*,",
+      'STORM': "metrics/api/cluster/summary,"
     };
     var services = App.cache['services'];
     services.forEach(function (service) {

ambari-web/app/mappers/server_data_mapper.js

@@ -111,6 +111,7 @@ App.QuickDataMapper = App.ServerDataMapper.extend({
   getJsonProperty: function (json, path) {
     var pathArr = path.split('.');
     var current = json;
+    pathArr = this.filterDotted(pathArr);
     while (pathArr.length && current) {
       if (pathArr[0].substr(-1) == ']') {
         var index = parseInt(pathArr[0].substr(-2, 1));
@@ -126,6 +127,29 @@ App.QuickDataMapper = App.ServerDataMapper.extend({
     return current;
   },
 
+  filterDotted: function(arr) {
+    var buffer = [];
+    var dottedBuffer = [];
+    var dotted = false;
+    arr.forEach(function(item) {
+      if(/\['|\["/.test(item)) {
+        dottedBuffer.push(item.substr(2, item.length));
+        dotted = true;
+      } else if (dotted && !/\]'|"\]/.test(item)) {
+        dottedBuffer.push(item);
+      } else if (/']|"]/.test(item)) {
+        dottedBuffer.push(item.substr(0, item.length - 2));
+        buffer.push(dottedBuffer.join('.'));
+        dotted = false;
+        dottedBuffer = [];
+      } else {
+        buffer.push(item);
+      }
+    });
+
+    return buffer;
+  },
+
   /**
    * properly delete record from model
    * @param item

ambari-web/app/mappers/service_metrics_mapper.js

@@ -126,6 +126,15 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
     heap_memory_used: 'masterComponent.ServiceComponentInfo.HeapMemoryUsed',
     heap_memory_max: 'masterComponent.ServiceComponentInfo.HeapMemoryMax'
   },
+  stormConfig: {
+    total_tasks: 'restApiComponent.metrics.api.cluster.summary.["tasks.total"]',
+    total_slots: 'restApiComponent.metrics.api.cluster.summary.["slots.total"]',
+    free_slots: 'restApiComponent.metrics.api.cluster.summary.["slots.free"]',
+    used_slots: 'restApiComponent.metrics.api.cluster.summary.["tasks.total"]',
+    topologies: 'restApiComponent.metrics.api.cluster.summary.topologies',
+    total_executors: 'restApiComponent.metrics.api.cluster.summary.["executors.total"]',
+    nimbus_uptime: 'restApiComponent.metrics.api.cluster.summary.["nimbus.uptime"]'
+  },
 
   model3: App.HostComponent,
   config3: {
@@ -224,6 +233,12 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.MapReduce2Service, finalJson);
+        } else if (item && item.ServiceInfo && item.ServiceInfo.service_name == "STORM") {
+          finalJson = this.stormMapper(item);
+          finalJson.rand = Math.random();
+          this.mapQuickLinks(finalJson, item);
+          result.push(finalJson);
+          App.store.load(App.StormService, finalJson);
         } else {
           finalJson = this.parseIt(item, this.config);
           finalJson.rand = Math.random();
@@ -600,5 +615,20 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
       }
     });
     return finalJson;
+  },
+
+  /**
+   * Storm mapper
+   */
+  stormMapper: function(item) {
+    var finalConfig = jQuery.extend({}, this.config);
+    var stormConfig = this.stormConfig;
+    item.components.forEach(function(component) {
+      if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name == "STORM_REST_API") {
+        item.restApiComponent = component;
+        finalConfig = jQuery.extend({}, finalConfig, stormConfig);
+      }
+    });
+    return this.parseIt(item, finalConfig);
   }
 });

ambari-web/app/messages.js

@@ -279,6 +279,17 @@ Em.I18n.translations = {
   'services.tez.description':'Tez is the next generation Hadoop Query Processing framework written on top of YARN',
   'services.falcon.description': 'Falcon mirroring engine',
   'services.storm.description': 'Apache Hadoop Stream processing framework',
+  'services.storm.slots.metrics.title': 'Number of slots',
+  'services.storm.slots.metrics.free': 'Free slots',
+  'services.storm.slots.metrics.total': 'Total slots',
+  'services.storm.slots.metrics.used': 'Used slots',
+  'services.storm.executors.metrics.title': 'Number of executors',
+  'services.storm.executors.metrics.total': 'Total executors',
+  'services.storm.topology.metrics.title': 'Number of topologies',
+  'services.storm.topology.metrics.total': 'Total topologies',
+  'services.storm.tasks.metrics.title': 'Number of tasks',
+  'services.storm.tasks.metrics.total': 'Total tasks',
+
 
   'services.alerts.head':'You have {0} critical alert notification(s).',
   'services.alerts.OK.timePrefix': 'OK for ',

ambari-web/app/mixins/common/chart/storm_linear_time.js

@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+var App = require('app');
+
+App.StormLinearTimeChartMixin = Em.Mixin.create({
+  chartStormModel: function() {
+    return App.StormService.find().objectAt(0);
+  }.property(''),
+
+  loadData: function() {
+    this._refreshGraph();
+  },
+
+  showGraphInPopup: function() {
+    this._super();
+    var self = this;
+    // show data in popup
+    Em.run.next(function() {
+      self.set('isPopupReady', true);
+    });
+  },
+
+  transformToSeries: function () {
+    return this.getMappedStormData();
+  },
+
+  getMappedStormData: function() {
+    var seriesData = [];
+    this.get('stormChartDefinition').forEach(function(item) {
+      seriesData.push(this.transformData(this.get('chartStormModel.' + item.value), item.name));
+    }, this);
+    return seriesData
+  }
+});

ambari-web/app/models.js

@@ -34,6 +34,7 @@ require('models/service/mapreduce');
 require('models/service/mapreduce2');
 require('models/service/hbase');
 require('models/service/flume');
+require('models/service/storm');
 require('models/alert');
 require('models/user');
 require('models/host');

ambari-web/app/models/service/storm.js

@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+
+App.StormService = App.Service.extend({
+  version: DS.attr('string'),
+  totalTasks: DS.attr('number'),
+  totalSlots: DS.attr('number'),
+  usedSlots: DS.attr('number'),
+  freeSlots: DS.attr('number'),
+  totalExecutors: DS.attr('number'),
+  topologies: DS.attr('number'),
+  nimbusUptime: DS.attr('number')
+});
+
+App.StormService.FIXTURES = [];

ambari-web/app/views.js

@@ -203,6 +203,11 @@ require('views/main/service/info/metrics/flume/gc');
 require('views/main/service/info/metrics/flume/jvm_heap');
 require('views/main/service/info/metrics/flume/jvm_threads_runnable');
 require('views/main/service/info/metrics/flume/cpu_user');
+require('views/main/service/info/metrics/storm/slots_number_metric');
+require('views/main/service/info/metrics/storm/executors_metric');
+require('views/main/service/info/metrics/storm/tasks_metric');
+require('views/main/service/info/metrics/storm/topologies_metric');
+
 require('views/main/service/add_view');
 require('views/main/service/reassign_view');
 require('views/main/service/reassign/step1_view');

ambari-web/app/views/main/service/info/metrics/storm/executors_metric.js

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('mixins/common/chart/storm_linear_time');
+
+App.ChartServiceMetricsSTORM_Executors = App.ChartLinearTimeView.extend(App.StormLinearTimeChartMixin, {
+  id: "service-metrics-storm-executors",
+  title: Em.I18n.t('services.storm.executors.metrics.title'),
+  renderer: 'line',
+  yAxisFormatter: App.ChartLinearTimeView.DefaultFormatter,
+
+  stormChartDefinition: [
+    {
+      name: Em.I18n.t('services.storm.executors.metrics.total'),
+      value: 'totalExecutors'
+    }
+  ]
+
+});

ambari-web/app/views/main/service/info/metrics/storm/slots_number_metric.js

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('mixins/common/chart/storm_linear_time');
+
+App.ChartServiceMetricsSTORM_SlotsNumber = App.ChartLinearTimeView.extend(App.StormLinearTimeChartMixin, {
+  id: "service-metrics-storm-supervisor-allocated",
+  title: Em.I18n.t('services.storm.slots.metrics.title'),
+  renderer: 'line',
+  yAxisFormatter: App.ChartLinearTimeView.DefaultFormatter,
+
+  stormChartDefinition: [
+    {
+      name: Em.I18n.t('services.storm.slots.metrics.total'),
+      value: 'totalSlots'
+    },
+    {
+      name: Em.I18n.t('services.storm.slots.metrics.free'),
+      value: 'freeSlots'
+    },
+    {
+      name: Em.I18n.t('services.storm.slots.metrics.used'),
+      value: 'usedSlots'
+    }
+  ]
+
+});

ambari-web/app/views/main/service/info/metrics/storm/tasks_metric.js

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('mixins/common/chart/storm_linear_time');
+
+App.ChartServiceMetricsSTORM_Tasks = App.ChartLinearTimeView.extend(App.StormLinearTimeChartMixin, {
+  id: "service-metrics-storm-tasks",
+  title: Em.I18n.t('services.storm.tasks.metrics.title'),
+  renderer: 'line',
+  yAxisFormatter: App.ChartLinearTimeView.DefaultFormatter,
+
+  stormChartDefinition: [
+    {
+      name: Em.I18n.t('services.storm.tasks.metrics.total'),
+      value: 'totalTasks'
+    }
+  ]
+
+});

ambari-web/app/views/main/service/info/metrics/storm/topologies_metric.js

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('mixins/common/chart/storm_linear_time');
+
+App.ChartServiceMetricsSTORM_Topologies = App.ChartLinearTimeView.extend(App.StormLinearTimeChartMixin, {
+  id: "service-metrics-storm-topologies",
+  title: Em.I18n.t('services.storm.topology.metrics.title'),
+  renderer: 'line',
+  yAxisFormatter: App.ChartLinearTimeView.DefaultFormatter,
+
+  stormChartDefinition: [
+    {
+      name: Em.I18n.t('services.storm.topology.metrics.total'),
+      value: 'topologies'
+    }
+  ]
+
+});

ambari-web/app/views/main/service/info/summary.js

@@ -366,6 +366,16 @@ App.MainServiceInfoSummaryView = Em.View.extend({
               //App.ChartServiceMetricsFlume_JVMThreadsRunnable.extend(),
               App.ChartServiceMetricsFlume_CPUUser.extend()]];
           break;
+        case 'storm':
+          graphs = [
+            [
+              App.ChartServiceMetricsSTORM_SlotsNumber.extend(),
+              App.ChartServiceMetricsSTORM_Executors.extend(),
+              App.ChartServiceMetricsSTORM_Topologies.extend(),
+              App.ChartServiceMetricsSTORM_Tasks.extend()
+            ]
+          ];
+          break;
         default:
           break;
       }

ambari-web/test/mappers/server_data_mapper_test.js

@@ -41,7 +41,10 @@ describe('App.QuickDataMapper', function () {
         }
       ]
     },
-    a2: 'val3'
+    a2: 'val3',
+    item: {
+      'key.dotted': 'val6'
+    }
   };
 
   describe('#getJsonProperty', function() {
@@ -70,7 +73,8 @@ describe('App.QuickDataMapper', function () {
       f4_type: 'array',
       f4: {
         item: 'c2'
-      }
+      },
+      f5: 'item.["key.dotted"]'
     };
     var mapper = App.QuickDataMapper.create();
     var result = mapper.parseIt(test_json, config);
@@ -89,6 +93,9 @@ describe('App.QuickDataMapper', function () {
     it('Generate array of json fields', function() {
       expect(result.f4).to.eql(['val1','val4','val5']);
     });
+    it('Check value with dotted key', function() {
+      expect(result.f5).to.eql('val6');
+    });
   });
 
 });