AMBARI-11992 Hosts page responsiveness is severely impacted when metrics API responses are slow. (atkach)

ambari-web/app/controllers/global/update_controller.js

@@ -41,6 +41,11 @@ App.UpdateController = Em.Controller.extend({
 
   paginationKeys: ['page_size', 'from'],
 
+  /**
+   * @type {string}
+   */
+  HOSTS_TEST_URL: '/data/hosts/HDP2/hosts.json',
+
   /**
    * map which track status of requests, whether it's running or completed
    * @type {object}
@@ -55,23 +60,19 @@ App.UpdateController = Em.Controller.extend({
 
   /**
    * construct URL from real URL and query parameters
-   * @param testUrl
    * @param realUrl
    * @param queryParams
    * @return {String}
    */
-  getComplexUrl: function (testUrl, realUrl, queryParams) {
+  getComplexUrl: function (realUrl, queryParams) {
     var prefix = App.get('apiPrefix') + '/clusters/' + App.get('clusterName'),
       params = '';
 
-    if (App.get('testMode')) {
-      return testUrl;
-    } else {
-      if (queryParams) {
-        params = this.computeParameters(queryParams);
-      }
-      return prefix + realUrl.replace('<parameters>', params);
+    if (queryParams) {
+      params = this.computeParameters(queryParams);
     }
+    params = (params.length > 0) ? params + "&" : params;
+    return prefix + realUrl.replace('<parameters>', params);
   },
 
   /**
@@ -111,7 +112,7 @@ App.UpdateController = Em.Controller.extend({
       }
       params += '&';
     });
-    return params;
+    return params.substring(0, params.length - 1);
   },
 
   /**
@@ -172,28 +173,44 @@ App.UpdateController = Em.Controller.extend({
     }
   },
 
-  updateHost: function (callback, error) {
-    var testUrl = '/data/hosts/HDP2/hosts.json',
+  /**
+   *
+   * @param {Function} callback
+   * @param {Function} error
+   * @param {boolean} lazyLoadMetrics
+   */
+  updateHost: function (callback, error, lazyLoadMetrics) {
+    var testUrl = this.get('HOSTS_TEST_URL'),
       self = this,
-      hostDetailsFilter = '';
-    var realUrl = '/hosts?<parameters>fields=Hosts/rack_info,Hosts/host_name,Hosts/maintenance_state,Hosts/public_host_name,Hosts/cpu_count,Hosts/ph_cpu_count,' +
+      hostDetailsFilter = '',
+      realUrl = '/hosts?fields=Hosts/rack_info,Hosts/host_name,Hosts/maintenance_state,Hosts/public_host_name,Hosts/cpu_count,Hosts/ph_cpu_count,' +
       'alerts_summary,Hosts/host_status,Hosts/last_heartbeat_time,Hosts/ip,host_components/HostRoles/state,host_components/HostRoles/maintenance_state,' +
       'host_components/HostRoles/stale_configs,host_components/HostRoles/service_name,host_components/HostRoles/desired_admin_state,' +
-        'metrics/disk,metrics/load/load_one,Hosts/total_mem<hostAuxiliaryInfo><stackVersions>&minimal_response=true';
-    var hostAuxiliaryInfo = ',Hosts/os_arch,Hosts/os_type,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free';
-    var stackVersionInfo = ',stack_versions/HostStackVersions,' +
+        '<metrics>Hosts/total_mem<hostDetailsParams><stackVersions>&minimal_response=true',
+      hostDetailsParams = ',Hosts/os_arch,Hosts/os_type,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free',
+      stackVersionInfo = ',stack_versions/HostStackVersions,' +
       'stack_versions/repository_versions/RepositoryVersions/repository_version,stack_versions/repository_versions/RepositoryVersions/id,' +
-      'stack_versions/repository_versions/RepositoryVersions/display_name';
-    realUrl = realUrl.replace("<stackVersions>", (App.get('supports.stackUpgrade') ? stackVersionInfo : ""));
+      'stack_versions/repository_versions/RepositoryVersions/display_name',
+      mainHostController = App.router.get('mainHostController'),
+      sortProperties = mainHostController.getSortProps();
 
-    if (App.router.get('currentState.name') == 'index' && App.router.get('currentState.parentState.name') == 'hosts') {
+    if (App.router.get('currentState.parentState.name') == 'hosts') {
       App.updater.updateInterval('updateHost', App.get('contentUpdateInterval'));
+      hostDetailsParams = '';
+      this.get('queryParams').set('Hosts', mainHostController.getQueryParameters(true));
     }
     else {
-      if (App.router.get('currentState.parentState.name') == 'hostDetails' &&
-          ['summary', 'alerts', 'stackVersions'].contains(App.router.get('currentState.name'))) {
+      if (App.router.get('currentState.parentState.name') == 'hostDetails') {
         hostDetailsFilter = App.router.get('location.lastSetURL').match(/\/hosts\/(.*)\/(summary|alerts|stackVersions)/)[1];
         App.updater.updateInterval('updateHost', App.get('componentsUpdateInterval'));
+        //if host details page opened then request info only of one displayed host
+        this.get('queryParams').set('Hosts', [
+          {
+            key: 'Hosts/host_name',
+            value: [hostDetailsFilter],
+            type: 'MULTIPLE'
+          }
+        ]);
       }
       else {
         callback();
@@ -203,24 +220,18 @@ App.UpdateController = Em.Controller.extend({
         }
       }
     }
-    var mainHostController = App.router.get('mainHostController'),
-      sortProperties = mainHostController.getSortProps();
-    if (hostDetailsFilter) {
-      //if host details page opened then request info only of one displayed host
-      this.get('queryParams').set('Hosts', [
-        {
-          key: 'Hosts/host_name',
-          value: [hostDetailsFilter],
-          type: 'MULTIPLE'
-        }
-      ]);
-    } else {
-      hostAuxiliaryInfo = '';
-      this.get('queryParams').set('Hosts', mainHostController.getQueryParameters(true));
-    }
-    realUrl = realUrl.replace('<hostAuxiliaryInfo>', hostAuxiliaryInfo);
+
+    realUrl = realUrl.replace("<stackVersions>", (App.get('supports.stackUpgrade') ? stackVersionInfo : ""));
+    realUrl = realUrl.replace("<metrics>", (lazyLoadMetrics ? "" : "metrics/disk,metrics/load/load_one,"));
+    realUrl = realUrl.replace('<hostDetailsParams>', hostDetailsParams);
 
     var clientCallback = function (skipCall, queryParams) {
+      var completeCallback = function () {
+        callback();
+        if (lazyLoadMetrics) {
+          self.loadHostsMetric(queryParams);
+        }
+      };
       if (skipCall) {
         //no hosts match filter by component
         App.hostsMapper.map({
@@ -230,35 +241,18 @@ App.UpdateController = Em.Controller.extend({
         callback();
       }
       else {
-        var params = self.computeParameters(queryParams),
-          paginationProps = self.computeParameters(queryParams.filter(function (param) {
-            return (this.get('paginationKeys').contains(param.key));
-          }, self)),
-          sortProps = self.computeParameters(sortProperties);
-
-        if ((params.length + paginationProps.length + sortProps.length) > 0) {
-          realUrl = App.get('apiPrefix') + '/clusters/' + App.get('clusterName') +
-            realUrl.replace('<parameters>', '') +
-            (paginationProps.length > 0 ? '&' + paginationProps.substring(0, paginationProps.length - 1) : '') +
-            (sortProps.length > 0 ? '&' + sortProps.substring(0, sortProps.length - 1) : '');
-          if (App.get('testMode')) {
-            realUrl = testUrl;
-          }
-          App.HttpClient.get(realUrl, App.hostsMapper, {
-            complete: callback,
-            doGetAsPost: true,
-            params: params.substring(0, params.length - 1),
-            error: error
-          });
-        }
-        else {
-          var hostsUrl = self.getComplexUrl(testUrl, realUrl, queryParams);
-          App.HttpClient.get(hostsUrl, App.hostsMapper, {
-            complete: callback,
-            doGetAsPost: false,
-            error: error
-          });
+        if (App.get('testMode')) {
+          realUrl = testUrl;
+        } else {
+          realUrl = self.addParamsToHostsUrl.call(self, queryParams, sortProperties, realUrl);
         }
+
+        App.HttpClient.get(realUrl, App.hostsMapper, {
+          complete: completeCallback,
+          doGetAsPost: true,
+          params: self.computeParameters(queryParams),
+          error: error
+        });
       }
     };
 
@@ -267,6 +261,54 @@ App.UpdateController = Em.Controller.extend({
     }
   },
 
+  /**
+   *
+   * @param {Array} queryParams
+   * @param {Array} sortProperties
+   * @param {string} realUrl
+   * @returns {string}
+   */
+  addParamsToHostsUrl: function (queryParams, sortProperties, realUrl) {
+    var paginationProps = this.computeParameters(queryParams.filter(function (param) {
+      return (this.get('paginationKeys').contains(param.key));
+    }, this));
+    var sortProps = this.computeParameters(sortProperties);
+
+    return App.get('apiPrefix') + '/clusters/' + App.get('clusterName') + realUrl +
+      (paginationProps.length > 0 ? '&' + paginationProps : '') +
+      (sortProps.length > 0 ? '&' + sortProps : '');
+  },
+
+  /**
+   * lazy load metrics of hosts
+   * @param {Array} queryParams
+   * @returns {$.ajax|null}
+   */
+  loadHostsMetric: function (queryParams) {
+    var realUrl = '/hosts?fields=metrics/disk/disk_free,metrics/disk/disk_total,metrics/load/load_one&minimal_response=true';
+
+    if (App.Service.find('AMBARI_METRICS').get('isStarted')) {
+      return App.ajax.send({
+        name: 'hosts.metrics.lazy_load',
+        sender: this,
+        data: {
+          url: this.addParamsToHostsUrl(queryParams, [], realUrl),
+          parameters: this.computeParameters(queryParams)
+        },
+        success: 'loadHostsMetricSuccessCallback'
+      });
+    }
+    return null;
+  },
+
+  /**
+   * success callback of <code>loadHostsMetric</code>
+   * @param {object} data
+   */
+  loadHostsMetricSuccessCallback: function (data) {
+    App.hostsMapper.setMetrics(data);
+  },
+
   /**
    * identify if any filter by host-component is active
    * if so run @getHostByHostComponents
@@ -291,14 +333,13 @@ App.UpdateController = Em.Controller.extend({
    * @param callback
    */
   getHostByHostComponents: function (callback) {
-    var testUrl = '/data/hosts/HDP2/hosts.json';
     var realUrl = '/hosts?<parameters>minimal_response=true';
 
     App.ajax.send({
       name: 'hosts.host_components.pre_load',
       sender: this,
       data: {
-        url: this.getComplexUrl(testUrl, realUrl, this.get('queryParams.Hosts')),
+        url: this.getComplexUrl(realUrl, this.get('queryParams.Hosts')),
         callback: callback
       },
       success: 'getHostByHostComponentsSuccessCallback',

ambari-web/app/controllers/main/dashboard/config_history_controller.js

@@ -226,6 +226,7 @@ App.MainConfigHistoryController = Em.ArrayController.extend(App.TableServerMixin
       if (queryParams) {
         params = App.router.get('updateController').computeParameters(queryParams);
       }
+      params = (params.length > 0) ? params + "&" : params;
       return this.get('realUrl').replace('<parameters>', params);
     }
   },

ambari-web/app/mappers/hosts_mapper.js

@@ -178,5 +178,20 @@ App.hostsMapper = App.QuickDataMapper.create({
       App.get('componentConfigMapper').addNewHostComponents(newHostComponentsMap, cacheServices);
     }
     console.timeEnd('App.hostsMapper execution time');
+  },
+
+  /**
+   * set metric fields of hosts
+   * @param {object} data
+   */
+  setMetrics: function (data) {
+    this.get('model').find().forEach(function (host) {
+      if (host.get('isRequested')) {
+        var hostMetrics = data.items.findProperty('Hosts.host_name', host.get('hostName'));
+        host.set('diskTotal', Em.get(hostMetrics, 'metrics.disk.disk_total'));
+        host.set('diskFree', Em.get(hostMetrics, 'metrics.disk.disk_free'));
+        host.set('loadOne', Em.get(hostMetrics, 'metrics.load.load_one'));
+      }
+    }, this);
   }
 });

ambari-web/app/utils/ajax/ajax.js

@@ -2293,6 +2293,22 @@ var urls = {
       }
     }
   },
+  'hosts.metrics.lazy_load': {
+    real: '',
+    mock: '/data/hosts/HDP2/hosts.json',
+    format: function (data) {
+      return {
+        url: data.url,
+        headers: {
+          'X-Http-Method-Override': 'GET'
+        },
+        type: 'POST',
+        data: JSON.stringify({
+          "RequestInfo": {"query": data.parameters}
+        })
+      }
+    }
+  },
   'hosts.bulk.operations': {
     real: '/clusters/{clusterName}/hosts?fields=Hosts/host_name,Hosts/maintenance_state,' +
     'host_components/HostRoles/state,host_components/HostRoles/maintenance_state,' +

ambari-web/app/views/main/host.js

@@ -83,7 +83,7 @@ App.MainHostView = App.TableView.extend(App.TableServerViewMixin, {
   refresh: function () {
     this.set('filteringComplete', false);
     var updaterMethodName = this.get('updater.tableUpdaterMap')[this.get('tableName')];
-    this.get('updater')[updaterMethodName](this.updaterSuccessCb.bind(this), this.updaterErrorCb.bind(this));
+    this.get('updater')[updaterMethodName](this.updaterSuccessCb.bind(this), this.updaterErrorCb.bind(this), true);
     return true;
   },
 
@@ -384,7 +384,7 @@ App.MainHostView = App.TableView.extend(App.TableServerViewMixin, {
       name: 'hosts.bulk.operations',
       sender: this,
       data: {
-        parameters: params.substring(0, params.length - 1),
+        parameters: params,
         operationData: operationData,
         loadingPopup: loadingPopup
       },

ambari-web/app/views/main/host/details.js

@@ -59,7 +59,7 @@ App.MainHostDetailsView = Em.View.extend({
   didInsertElement: function() {
     var self = this;
 
-    this.set('isLoaded', false);
+    this.set('isLoaded', App.Host.find(this.get('content.id')).get('isLoaded'));
     App.router.get('updateController').updateHost(function () {
       self.set('isLoaded', true);
       if (!self.get('content.isLoaded')) {

ambari-web/test/controllers/global/update_controller_test.js

@@ -228,4 +228,80 @@ describe('App.UpdateController', function () {
       });
     });
   });
+
+  describe("#getComplexUrl()", function () {
+    beforeEach(function () {
+      sinon.stub(App, 'get').returns('mock');
+      sinon.stub(controller, 'computeParameters').returns('params');
+    });
+    afterEach(function () {
+      App.get.restore();
+      controller.computeParameters.restore();
+    });
+    it("queryParams is empty", function () {
+      expect(controller.getComplexUrl('<parameters>')).to.equal('mock/clusters/mock');
+    });
+    it("queryParams is present", function () {
+      var queryParams = [
+        {
+          type: "EQUAL",
+          key: "key",
+          value: "value"
+        }
+      ];
+      expect(controller.getComplexUrl('<parameters>', queryParams)).to.equal('mock/clusters/mockparams&');
+    });
+  });
+
+  describe("#addParamsToHostsUrl()", function () {
+    beforeEach(function () {
+      sinon.stub(App, 'get').returns('mock');
+      sinon.stub(controller, 'computeParameters').returns('params');
+    });
+    afterEach(function () {
+      App.get.restore();
+      controller.computeParameters.restore();
+    });
+    it("", function () {
+      expect(controller.addParamsToHostsUrl([], [], 'url')).to.equal('mock/clusters/mockurl&params&params');
+    });
+  });
+
+  describe("#loadHostsMetric()", function () {
+    beforeEach(function () {
+      this.mock = sinon.stub(App.Service, 'find');
+      sinon.stub(controller, 'computeParameters');
+      sinon.stub(controller, 'addParamsToHostsUrl');
+      sinon.stub(App.ajax, 'send');
+    });
+    afterEach(function () {
+      App.Service.find.restore();
+      controller.computeParameters.restore();
+      controller.addParamsToHostsUrl.restore();
+      App.ajax.send.restore();
+    });
+    it("AMBARI_METRICS is not started", function () {
+      this.mock.returns(Em.Object.create({isStarted: false}));
+      expect(controller.loadHostsMetric([])).to.be.null;
+      expect(App.ajax.send.called).to.be.false;
+    });
+    it("AMBARI_METRICS is started", function () {
+      this.mock.returns(Em.Object.create({isStarted: true}));
+      expect(controller.loadHostsMetric([])).to.be.object;
+      expect(App.ajax.send.calledOnce).to.be.true;
+    });
+  });
+
+  describe("#loadHostsMetricSuccessCallback()", function () {
+    beforeEach(function () {
+      sinon.stub(App.hostsMapper, 'setMetrics');
+    });
+    afterEach(function () {
+      App.hostsMapper.setMetrics.restore();
+    });
+    it("", function () {
+      controller.loadHostsMetricSuccessCallback({});
+      expect(App.hostsMapper.setMetrics.calledWith({})).to.be.true;
+    });
+  });
 });

ambari-web/test/controllers/main/dashboard/config_history_controller_test.js

@@ -117,7 +117,7 @@ describe('MainConfigHistoryController', function () {
         if ('testMode' === k) return false;
         return Em.get(App, k);
       });
-      expect(controller.getUrl({})).to.equal('/api/v1/clusters/mycluster/configurations/service_config_versions?paramsfields=service_config_version,user,group_id,group_name,is_current,createtime,service_name,hosts,service_config_version_note,is_cluster_compatible&minimal_response=true');
+      expect(controller.getUrl({})).to.equal('/api/v1/clusters/mycluster/configurations/service_config_versions?params&fields=service_config_version,user,group_id,group_name,is_current,createtime,service_name,hosts,service_config_version_note,is_cluster_compatible&minimal_response=true');
     });
   });
 

ambari-web/test/mappers/hosts_mapper_test.js

@@ -25,7 +25,55 @@ require('mappers/server_data_mapper');
 require('mappers/hosts_mapper');
 
 describe('App.hostsMapper', function () {
+  var mapper = App.hostsMapper;
 
-
-
+  describe("#setMetrics()", function() {
+    var data = {
+      items: [
+        {
+          Hosts: {
+            host_name: 'host1'
+          },
+          metrics: {
+            load: {
+              load_one: 1
+            }
+          }
+        }
+      ]
+    };
+    beforeEach(function(){
+      this.mock = sinon.stub(App.Host, 'find')
+    });
+    afterEach(function(){
+      this.mock.restore();
+    });
+    it("Host not in the model", function() {
+      var host = Em.Object.create({
+        hostName: 'host2',
+        isRequested: true
+      });
+      this.mock.returns([host]);
+      mapper.setMetrics(data);
+      expect(host.get('loadOne')).to.be.undefined;
+    });
+    it("Host not in the filter", function() {
+      var host = Em.Object.create({
+        hostName: 'host1',
+        isRequested: false
+      });
+      this.mock.returns([host]);
+      mapper.setMetrics(data);
+      expect(host.get('loadOne')).to.be.undefined;
+    });
+    it("Host should have updated metrics", function() {
+      var host = Em.Object.create({
+        hostName: 'host1',
+        isRequested: true
+      });
+      this.mock.returns([host]);
+      mapper.setMetrics(data);
+      expect(host.get('loadOne')).to.equal(1);
+    });
+  });
 });