AMBARI-11781 Perf: Combine metrics requests on UI Host page. (atkach)

ambari-web/app/utils/ajax/ajax.js

@@ -994,6 +994,11 @@ var urls = {
     'mock': '/data/services/metrics/ambari_metrics/regionserver_compaction_queue_size.json',
     'testInProduction': true
   },
+  'service.metrics.ambari_metrics.aggregated': {
+    'real': '/clusters/{clusterName}/services/AMBARI_METRICS/components/METRICS_COLLECTOR?fields={fields}',
+    'mock': '/data/services/metrics/ambari_metrics/master_average_load.json',
+    'testInProduction': true
+  },
   'service.metrics.hdfs.block_status': {
     'real': '/clusters/{clusterName}/hosts/{nameNodeName}/host_components/NAMENODE?fields=metrics/dfs/FSNamesystem/PendingReplicationBlocks[{fromSeconds},{toSeconds},{stepSeconds}],metrics/dfs/FSNamesystem/UnderReplicatedBlocks[{fromSeconds},{toSeconds},{stepSeconds}]',
     'mock': '/data/services/metrics/hdfs/block_status.json',
@@ -1146,6 +1151,10 @@ var urls = {
     'mock': '/data/cluster_metrics/network_1hr.json',
     'testInProduction': true
   },
+  'host.metrics.aggregated': {
+    'real': '/clusters/{clusterName}/hosts/{hostName}?fields={fields}',
+    'mock': '/data/hosts/metrics/cpu.json'
+  },
   'host.metrics.cpu': {
     'real': '/clusters/{clusterName}/hosts/{hostName}?fields=metrics/cpu/cpu_user[{fromSeconds},{toSeconds},{stepSeconds}],metrics/cpu/cpu_wio[{fromSeconds},{toSeconds},{stepSeconds}],metrics/cpu/cpu_nice[{fromSeconds},{toSeconds},{stepSeconds}],metrics/cpu/cpu_aidle[{fromSeconds},{toSeconds},{stepSeconds}],metrics/cpu/cpu_system[{fromSeconds},{toSeconds},{stepSeconds}],metrics/cpu/cpu_idle[{fromSeconds},{toSeconds},{stepSeconds}]',
     'mock': '/data/hosts/metrics/cpu.json',

ambari-web/app/views/common/chart/linear_time.js

@@ -169,13 +169,17 @@ App.ChartLinearTimeView = Ember.View.extend({
   },
 
   loadData: function() {
-    App.ajax.send({
-      name: this.get('ajaxIndex'),
-      sender: this,
-      data: this.getDataForAjaxRequest(),
-      success: '_refreshGraph',
-      error: 'loadDataErrorCallback'
-    });
+    if (this.get('loadGroup')) {
+      App.ChartLinearTimeView.LoadAggregator.add(this, this.get('loadGroup'));
+    } else {
+      App.ajax.send({
+        name: this.get('ajaxIndex'),
+        sender: this,
+        data: this.getDataForAjaxRequest(),
+        success: '_refreshGraph',
+        error: 'loadDataErrorCallback'
+      });
+    }
   },
 
   getDataForAjaxRequest: function() {
@@ -1030,4 +1034,140 @@ Rickshaw.Graph.Renderer.Stack.prototype.seriesPathFactory = function() {
     .y1( function(d) { return graph.y(d.y + d.y0) } )
     .defined(function(d) { return d.y!=null; })
     .interpolate(this.graph.interpolation).tension(this.tension);
-};
+};
+
+
+/**
+ * aggregate requests to load metrics by component name
+ * requests can be added via add method
+ * input example:
+ * {
+ *   data: request,
+ *   context: this,
+ *   startCallName: this.getServiceComponentMetrics,
+ *   successCallback: this.getMetricsSuccessCallback,
+ *   completeCallback: function () {
+ *     requestCounter--;
+ *     if (requestCounter === 0) this.onMetricsLoaded();
+ *   }
+ * }
+ * @type {Em.Object}
+ */
+App.ChartLinearTimeView.LoadAggregator = Em.Object.create({
+  /**
+   * @type {Array}
+   */
+  requests: [],
+
+  /**
+   * @type {number|null}
+   */
+  timeoutId: null,
+
+  /**
+   * time interval within which calls get collected
+   * @type {number}
+   * @const
+   */
+  BULK_INTERVAL: 1000,
+
+  /**
+   * add request
+   * every {{BULK_INTERVAL}} requests get collected, aggregated and sent to server
+   *
+   * @param {object} context
+   * @param {object} requestData
+   */
+  add: function (context, requestData) {
+    var self = this;
+
+    requestData.context = context;
+    this.get('requests').push(requestData);
+    if (Em.isNone(this.get('timeoutId'))) {
+      this.set('timeoutId', window.setTimeout(function () {
+        self.runRequests(self.get('requests'));
+        self.get('requests').clear();
+        clearTimeout(self.get('timeoutId'));
+        self.set('timeoutId', null);
+      }, this.get('BULK_INTERVAL')));
+    }
+  },
+
+  /**
+   * return requests which grouped into bulks
+   * @param {Array} requests
+   * @returns {object} bulks
+   */
+  groupRequests: function (requests) {
+    var bulks = {};
+
+    requests.forEach(function (request) {
+      var id = request.name;
+
+      if (Em.isNone(bulks[id])) {
+        bulks[id] = {
+          name: request.name,
+          fields: request.fields,
+          context: request.context
+        };
+        bulks[id].subRequests = [{
+          context: request.context
+        }];
+      } else {
+        bulks[id].fields.pushObjects(request.fields);
+        bulks[id].subRequests.push({
+          context: request.context
+        });
+      }
+    }, this);
+    return bulks;
+  },
+
+  /**
+   * run aggregated requests
+   * @param {Array} requests
+   */
+  runRequests: function (requests) {
+    var bulks = this.groupRequests(requests);
+    var self = this;
+
+    for (var id in bulks) {
+      (function (_request) {
+        var fields = self.formatRequestData(_request);
+        var hostName = (_request.context.get('content')) ? _request.context.get('content.hostName') : "";
+
+        App.ajax.send({
+          name: _request.name,
+          sender: _request.context,
+          data: {
+            fields: fields,
+            hostName: hostName
+          }
+        }).done(function (response) {
+          _request.subRequests.forEach(function (subRequest) {
+            subRequest.context._refreshGraph.call(subRequest.context, response);
+          }, this);
+        }).fail(function (jqXHR, textStatus, errorThrown) {
+          _request.subRequests.forEach(function (subRequest) {
+            subRequest.context.loadDataErrorCallback.call(subRequest.context, jqXHR, textStatus, errorThrown );
+          }, this);
+        });
+      })(bulks[id]);
+    }
+  },
+
+  /**
+   *
+   * @param {object} request
+   * @returns {number[]}
+   */
+  formatRequestData: function (request) {
+    var toSeconds = Math.round(App.dateTime() / 1000);
+    var timeUnit = request.context.get('timeUnitSeconds');
+    var fields = request.fields.uniq().map(function (field) {
+      return field + "[" + (toSeconds - timeUnit) + "," + toSeconds + "," + 15 + "]";
+    });
+
+    return fields.join(",");
+  }
+});

ambari-web/app/views/main/host/metrics/cpu.js

@@ -33,6 +33,11 @@ App.ChartHostMetricsCPU = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.cpu',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: ['metrics/cpu/cpu_user', 'metrics/cpu/cpu_wio', 'metrics/cpu/cpu_nice', 'metrics/cpu/cpu_aidle', 'metrics/cpu/cpu_system', 'metrics/cpu/cpu_idle']
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     if (jsonData && jsonData.metrics && jsonData.metrics.cpu) {

ambari-web/app/views/main/host/metrics/disk.js

@@ -34,6 +34,11 @@ App.ChartHostMetricsDisk = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.disk',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: ['metrics/disk/disk_total']
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     var GB = Math.pow(2, 30);

ambari-web/app/views/main/host/metrics/load.js

@@ -33,6 +33,11 @@ App.ChartHostMetricsLoad = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.load',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: ['metrics/load/load_fifteen', 'metrics/load/load_one', 'metrics/load/load_five']
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     if (jsonData && jsonData.metrics && jsonData.metrics.load) {

ambari-web/app/views/main/host/metrics/memory.js

@@ -34,6 +34,17 @@ App.ChartHostMetricsMemory = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.memory',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: [
+      'metrics/memory/swap_free',
+      'metrics/memory/mem_shared',
+      'metrics/memory/mem_free',
+      'metrics/memory/mem_cached',
+      'metrics/memory/mem_buffers'
+    ]
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     var KB = Math.pow(2, 10);

ambari-web/app/views/main/host/metrics/network.js

@@ -34,6 +34,16 @@ App.ChartHostMetricsNetwork = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.network',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: [
+      'metrics/network/bytes_in',
+      'metrics/network/bytes_out',
+      'metrics/network/pkts_in',
+      'metrics/network/pkts_out'
+    ]
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     if (jsonData && jsonData.metrics && jsonData.metrics.network) {

ambari-web/app/views/main/host/metrics/processes.js

@@ -33,6 +33,11 @@ App.ChartHostMetricsProcesses = App.ChartLinearTimeView.extend({
 
   ajaxIndex: 'host.metrics.processes',
 
+  loadGroup: {
+    name: 'host.metrics.aggregated',
+    fields: ['metrics/process/proc_total', 'metrics/process/proc_run']
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     if (jsonData && jsonData.metrics && jsonData.metrics.process) {

ambari-web/app/views/main/service/info/metrics/ambari_metrics/master_average_load.js

@@ -31,6 +31,11 @@ App.ChartServiceMetricsAMS_MasterAverageLoad = App.ChartLinearTimeView.extend({
   title: Em.I18n.t('services.service.info.metrics.ambariMetrics.master.averageLoad'),
   ajaxIndex: 'service.metrics.ambari_metrics.master.average_load',
 
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/master/AverageLoad']
+  },
+
   transformToSeries: function (jsonData) {
     var seriesArray = [];
     if (jsonData && jsonData.metrics && jsonData.metrics.hbase && jsonData.metrics.hbase.master) {

ambari-web/app/views/main/service/info/metrics/ambari_metrics/regionserver_block_cache_hit_percent.js

@@ -33,6 +33,11 @@ App.ChartServiceMetricsAMS_RegionServerBlockCacheHitPercent = App.ChartServiceMe
   yAxisFormatter: App.ChartLinearTimeView.PercentageFormatter,
   ajaxIndex: 'service.metrics.ambari_metrics.region_server.block_cache_hit_percent',
 
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/regionserver/blockCacheHitPercent']
+  },
+
   displayName: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.displayNames.blockCacheHitPercent'),
   regionServerName: 'blockCacheHitPercent'
 });

ambari-web/app/views/main/service/info/metrics/ambari_metrics/regionserver_compaction_queue_size.js

@@ -31,6 +31,11 @@ App.ChartServiceMetricsAMS_RegionServerCompactionQueueSize = App.ChartServiceMet
   title: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.compactionQueueSize'),
   ajaxIndex: 'service.metrics.ambari_metrics.region_server.compaction_queue_size',
 
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/regionserver/compactionQueueSize']
+  },
+
   displayName: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.displayNames.compactionQueueSize'),
   regionServerName: 'compactionQueueSize'
 });

ambari-web/app/views/main/service/info/metrics/ambari_metrics/regionserver_regions.js

@@ -31,6 +31,11 @@ App.ChartServiceMetricsAMS_RegionServerRegions = App.ChartServiceMetricsAMS_Regi
   title: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.regions'),
   ajaxIndex: 'service.metrics.ambari_metrics.region_server.regions',
 
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/regionserver/regions']
+  },
+
   displayName:  Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.displayNames.regionsCount'),
   regionServerName: 'regions'
 });

ambari-web/app/views/main/service/info/metrics/ambari_metrics/regionserver_requests.js

@@ -31,6 +31,11 @@ App.ChartServiceMetricsAMS_RegionServerRequests = App.ChartServiceMetricsAMS_Reg
   title: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.requests'),
   ajaxIndex: 'service.metrics.ambari_metrics.region_server.request',
 
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/regionserver/requests']
+  },
+
   displayName: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.displayNames.requestCount'),
   regionServerName: 'requests'
 });

ambari-web/app/views/main/service/info/metrics/ambari_metrics/regionserver_store_files.js

@@ -31,6 +31,12 @@ App.ChartServiceMetricsAMS_RegionServerStoreFiles = App.ChartServiceMetricsAMS_R
   title: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.storeFiles'),
   renderer: 'line',
   ajaxIndex: 'service.metrics.ambari_metrics.region_server.store_files',
+
+  loadGroup: {
+    name: 'service.metrics.ambari_metrics.aggregated',
+    fields: ['metrics/hbase/regionserver/storefiles']
+  },
+
   regionServerName: 'storefiles',
   displayName: Em.I18n.t('services.service.info.metrics.ambariMetrics.regionServer.displayNames.storeFilesCount')
 });

ambari-web/app/views/main/service/info/summary.js

@@ -296,9 +296,6 @@ App.MainServiceInfoSummaryView = Em.View.extend(App.UserPref, {
   rollingRestartStaleConfigSlaveComponents: function (componentName) {
     batchUtils.launchHostComponentRollingRestart(componentName.context, this.get('service.displayName'), this.get('service.passiveState') === "ON", true);
   },
-  /*
-   * 'Restart Required bar' ended
-   */
 
    /*
    * Find the graph class associated with the graph name, and split

ambari-web/test/views/common/chart/linear_time_test.js

@@ -209,3 +209,121 @@ describe('App.ChartLinearTimeView', function () {
     });
   });
 });
+
+
+describe('App.ChartLinearTimeView.LoadAggregator', function () {
+  var aggregator = App.ChartLinearTimeView.LoadAggregator;
+
+  describe("#add()", function () {
+    beforeEach(function () {
+      sinon.stub(window, 'setTimeout').returns('timeId');
+    });
+    afterEach(function () {
+      window.setTimeout.restore();
+    });
+    it("timeout started", function () {
+      aggregator.set('timeoutId', 'timeId');
+      aggregator.get('requests').clear();
+      aggregator.add({}, {});
+      expect(aggregator.get('requests')).to.not.be.empty;
+      expect(window.setTimeout.called).to.be.false;
+    });
+    it("timeout started", function () {
+      aggregator.set('timeoutId', null);
+      aggregator.get('requests').clear();
+      aggregator.add({}, {});
+      expect(aggregator.get('requests')).to.not.be.empty;
+      expect(window.setTimeout.calledOnce).to.be.true;
+      expect(aggregator.get('timeoutId')).to.equal('timeId');
+    });
+  });
+
+  describe("#groupRequests()", function () {
+    it("", function () {
+      var requests = [
+        {
+          name: 'r1',
+          context: 'c1',
+          fields: ['f1']
+        },
+        {
+          name: 'r2',
+          context: 'c2',
+          fields: ['f2']
+        },
+        {
+          name: 'r2',
+          context: 'c3',
+          fields: ['f3', 'f4']
+        }
+      ];
+      var result = aggregator.groupRequests(requests);
+
+      expect(result['r1'].subRequests.length).to.equal(1);
+      expect(result['r1'].fields.length).to.equal(1);
+      expect(result['r2'].subRequests.length).to.equal(2);
+      expect(result['r2'].fields.length).to.equal(3);
+    });
+  });
+
+  describe("#runRequests()", function () {
+    beforeEach(function () {
+      sinon.stub(aggregator, 'groupRequests', function (requests) {
+        return requests;
+      });
+      sinon.stub(aggregator, 'formatRequestData', function(_request){
+        return _request.fields;
+      });
+      sinon.stub(App.ajax, 'send', function(){
+        return {
+          done: Em.K,
+          fail: Em.K
+        }
+      });
+    });
+    afterEach(function () {
+      aggregator.groupRequests.restore();
+      App.ajax.send.restore();
+      aggregator.formatRequestData.restore();
+    });
+    it("", function () {
+      var context = Em.Object.create({content: {hostName: 'host1'}});
+      var requests = {
+        'r1': {
+          name: 'r1',
+          context: context,
+          fields: ['f3', 'f4']
+        }
+      };
+      aggregator.runRequests(requests);
+      expect(App.ajax.send.getCall(0).args[0]).to.eql({
+        name: 'r1',
+        sender: context,
+        data: {
+          fields: ['f3', 'f4'],
+          hostName: 'host1'
+        }
+      });
+    });
+  });
+
+  describe("#formatRequestData()", function () {
+    beforeEach(function () {
+      sinon.stub(App, 'dateTime').returns(4000000);
+
+    });
+    afterEach(function () {
+      App.dateTime.restore();
+
+    });
+    it("", function () {
+      var context = Em.Object.create({timeUnitSeconds: 3600});
+      var request = {
+        name: 'r1',
+        context: context,
+        fields: ['f3', 'f4']
+      };
+      expect(aggregator.formatRequestData(request)).to.equal('f3[400,4000,15],f4[400,4000,15]');
+    });
+  });
+});