AMBARI-3734 Reduce load on service mapper. (atkach)

ambari-web/app/controllers/global/cluster_controller.js

@@ -262,8 +262,7 @@ App.ClusterController = Em.Controller.extend({
    * @return {Boolean} Whether we have errors
    */
   loadUpdatedStatus: function(callback){
-    var location = App.router.get('location.lastSetURL');
-    if(!this.get('clusterName') || (this.get('isLoaded') && !(/\/main\/hosts.*/.test(location)))){
+    if(!this.get('clusterName')){
       callback();
       return false;
     }

ambari-web/app/controllers/global/update_controller.js

@@ -38,7 +38,7 @@ App.UpdateController = Em.Controller.extend({
   updateAll:function(){
     if(this.get('isWorking')) {
       App.updater.run(this, 'updateHostConditionally', 'isWorking');
-      App.updater.run(this, 'updateServiceMetric', 'isWorking', App.componentsUpdateInterval);
+      App.updater.run(this, 'updateServiceMetricConditionally', 'isWorking', App.componentsUpdateInterval);
       App.updater.run(this, 'graphsUpdate', 'isWorking');
       if (App.supports.hostOverrides) {
         App.updater.run(this, 'updateComponentConfig', 'isWorking');
@@ -61,6 +61,21 @@ App.UpdateController = Em.Controller.extend({
       callback();
     }
   },
+  /**
+   * Update service metrics depending on which page is open
+   * Make a call only on follow pages:
+   * /main/dashboard
+   * /main/services/*
+   * @param callback
+   */
+  updateServiceMetricConditionally: function(callback){
+    var location = App.router.get('location.lastSetURL');
+    if (/\/main\/(dashboard|services).*/.test(location)) {
+      this.updateServiceMetric(callback);
+    } else {
+      callback();
+    }
+  },
 
   updateHost:function(callback) {
     var testUrl = App.get('isHadoop2Stack') ? '/data/hosts/HDP2/hosts.json' : '/data/hosts/hosts.json';

ambari-web/app/mappers/server_data_mapper.js

@@ -22,7 +22,8 @@ var App = require('app');
 App.cache = {
   'Hosts': {},
   'previousHostStatuses': {},
-  'previousComponentStatuses': {}
+  'previousComponentStatuses': {},
+  'hostComponentsOnService': {}
 };
 
 App.ServerDataMapper = Em.Object.extend({

ambari-web/app/mappers/service_mapper.js

@@ -55,7 +55,9 @@ App.servicesMapper = App.QuickDataMapper.create({
     work_status: 'ServiceInfo.state',
     $rand: Math.random(),
     $alerts: [ 1, 2, 3 ],
-    host_components: 'host_components'
+    host_components: 'host_components',
+    running_host_components: 'running_host_components',
+    unknown_host_components: 'unknown_host_components'
   },
   hdfsConfig: {
     version: 'nameNodeComponent.host_components[0].metrics.dfs.namenode.Version',
@@ -64,7 +66,6 @@ App.servicesMapper = App.QuickDataMapper.create({
     active_name_node_id: 'active_name_node_id',
     standby_name_node_id: 'standby_name_node_id',
     standby_name_node2_id: 'standby_name_node2_id',
-    data_nodes: 'data_nodes',
     journal_nodes: 'journal_nodes',
     name_node_start_time: 'nameNodeComponent.host_components[0].metrics.runtime.StartTime',
     jvm_memory_heap_used: 'nameNodeComponent.host_components[0].metrics.jvm.memHeapUsedM',
@@ -88,9 +89,7 @@ App.servicesMapper = App.QuickDataMapper.create({
   yarnConfig: {
     version: 'resourceManagerComponent.ServiceComponentInfo.Version',
     resource_manager_node_id: 'resourceManagerComponent.host_components[0].HostRoles.host_name',
-    node_manager_nodes: 'node_manager_nodes',
     node_manager_live_nodes: 'node_manager_live_nodes',
-    yarn_client_nodes: 'yarn_client_nodes',
     resource_manager_start_time: 'resourceManagerComponent.ServiceComponentInfo.StartTime',
     jvm_memory_heap_used: 'resourceManagerComponent.host_components[0].metrics.jvm.memHeapUsedM',
     jvm_memory_heap_committed: 'resourceManagerComponent.host_components[0].metrics.jvm.memHeapCommittedM',
@@ -115,13 +114,11 @@ App.servicesMapper = App.QuickDataMapper.create({
   },
   mapReduce2Config: {
     version: 'jobHistoryServerComponent.ServiceComponentInfo.Version',
-    job_history_server_id: 'jobHistoryServerComponent.host_components[0].HostRoles.host_name',
-    map_reduce2_clients: 'map_reduce2_clients'
+    job_history_server_id: 'jobHistoryServerComponent.host_components[0].HostRoles.host_name'
   },
   mapReduceConfig: {
     version: 'jobTrackerComponent.ServiceComponentInfo.Version',
     job_tracker_id: 'jobTrackerComponent.host_components[0].HostRoles.host_name',
-    task_trackers: 'task_trackers',
     job_tracker_start_time: 'jobTrackerComponent.ServiceComponentInfo.StartTime',
     job_tracker_heap_used: 'jobTrackerComponent.ServiceComponentInfo.HeapMemoryUsed',
     job_tracker_heap_max: 'jobTrackerComponent.ServiceComponentInfo.HeapMemoryMax',
@@ -148,7 +145,6 @@ App.servicesMapper = App.QuickDataMapper.create({
   hbaseConfig: {
     version: 'masterComponent.ServiceComponentInfo.Version',
     master_id: 'masterComponent.host_components[0].HostRoles.host_name',
-    region_servers: 'region_servers',
     master_start_time: 'masterComponent.ServiceComponentInfo.MasterStartTime',
     master_active_time: 'masterComponent.ServiceComponentInfo.MasterActiveTime',
     average_load: 'masterComponent.ServiceComponentInfo.AverageLoad',
@@ -214,70 +210,27 @@ App.servicesMapper = App.QuickDataMapper.create({
 
       var result = [];
 
-      //establish relations between services and host-components
-      var serviceComponents = {},
-          masterComponents = [],
-          datanodeHosts = [],
-          taskTrackerHosts = [],
-          hbaseRegionserverHosts = [],
-          nodemanagerHosts = [],
-          yarnClientHosts = [],
-          mapreduce2ClientHosts = [];
-      App.HostComponent.find().forEach(function (hostComponent) {
-        if (serviceComponents[hostComponent.get('service.id')]) {
-          serviceComponents[hostComponent.get('service.id')].hostComponents.push(hostComponent.get('id'));
-        } else {
-          serviceComponents[hostComponent.get('service.id')] = {
-            hostComponents: [hostComponent.get('id')]
-          }
-        }
-
-        if (hostComponent.get('isMaster')) {
-          masterComponents.push(hostComponent);
-        }
-
-        switch (hostComponent.get('componentName')) {
-          case 'DATANODE':
-            datanodeHosts.push(hostComponent.get('host.id'));
-            break;
-          case 'TASKTRACKER':
-            taskTrackerHosts.push(hostComponent.get('host.id'));
-            break;
-          case 'HBASE_REGIONSERVER':
-            hbaseRegionserverHosts.push(hostComponent.get('host.id'));
-            break;
-          case 'NODEMANAGER':
-            nodemanagerHosts.push(hostComponent.get('host.id'));
-            break;
-          case 'YARN_CLIENT':
-            yarnClientHosts.push(hostComponent.get('host.id'));
-            break;
-          case 'MAPREDUCE2_CLIENT':
-            mapreduce2ClientHosts.push(hostComponent.get('host.id'));
-            break;
-        }
-      });
       //parse service metrics from components
       services.forEach(function (item) {
         var finalJson = [];
-        item.host_components = serviceComponents[item.ServiceInfo.service_name].hostComponents;
+        var serviceData = App.cache['hostComponentsOnService'][item.ServiceInfo.service_name];
+        item.host_components = serviceData.host_components;
+        item.running_host_components = serviceData.running_host_components;
+        item.unknown_host_components = serviceData.unknown_host_components;
         item.host_components.sort();
 
         if (item && item.ServiceInfo && item.ServiceInfo.service_name == "HDFS") {
           finalJson = this.hdfsMapper(item);
-          finalJson.data_nodes = datanodeHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.HDFSService, finalJson);
         } else if (item && item.ServiceInfo && item.ServiceInfo.service_name == "MAPREDUCE") {
           finalJson = this.mapreduceMapper(item);
-          finalJson.task_trackers = taskTrackerHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.MapReduceService, finalJson);
         } else if (item && item.ServiceInfo && item.ServiceInfo.service_name == "HBASE") {
           finalJson = this.hbaseMapper(item);
-          finalJson.region_servers = hbaseRegionserverHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.HBaseService, finalJson);
@@ -293,14 +246,11 @@ App.servicesMapper = App.QuickDataMapper.create({
           App.store.load(App.FlumeService, finalJson);
         } else if (item && item.ServiceInfo && item.ServiceInfo.service_name == "YARN") {
           finalJson = this.yarnMapper(item);
-          finalJson.node_manager_nodes = nodemanagerHosts;
-          finalJson.yarn_client_nodes = yarnClientHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.YARNService, finalJson);
         } else if (item && item.ServiceInfo && item.ServiceInfo.service_name == "MAPREDUCE2") {
           finalJson = this.mapreduce2Mapper(item);
-          finalJson.map_reduce2_clients = mapreduce2ClientHosts;
           finalJson.rand = Math.random();
           result.push(finalJson);
           App.store.load(App.MapReduce2Service, finalJson);
@@ -319,7 +269,7 @@ App.servicesMapper = App.QuickDataMapper.create({
 
       var servicesMap = {};
       //calculate service statuses according to their host-components
-      masterComponents.forEach(function (item) {
+      App.HostComponent.find().filterProperty('isMaster').forEach(function (item) {
         if (item) {
           this.countServiceComponents(item, servicesMap, servicesMap[item.get('service.id')]);
         }
@@ -348,8 +298,6 @@ App.servicesMapper = App.QuickDataMapper.create({
         isHbaseActive: false,
         serviceName: hostComponent.get('service.id'),
         isRunning: true,
-        runningHCs: [],
-        unknownHCs: [],
         hdfsHealthStatus: '',
         toolTipContent: ''
       };
@@ -435,9 +383,6 @@ App.servicesMapper = App.QuickDataMapper.create({
       hostComponent.get('workStatus') !== App.HostComponentStatus.unknown &&
       hostComponent.get('workStatus') !== App.HostComponentStatus.maintenance) {
       service.isRunning = false;
-      service.runningHCs.addObject(hostComponent);
-    } else if (hostComponent.get('workStatus') == App.HostComponentStatus.unknown) {
-      service.unknownHCs.addObject(hostComponent);
     }
   },
 
@@ -505,8 +450,6 @@ App.servicesMapper = App.QuickDataMapper.create({
         }
 
         _service.set('isStarted', service.everyStarted);
-        _service.set('runningHostComponents', service.runningHCs);
-        _service.set('unknownHostComponents', service.unknownHCs);
         _service.set('isStopped', service.isRunning);
         _service.set('toolTipContent', service.toolTipContent);
         if (serviceSpecificObj != null) {

ambari-web/app/mappers/status_mapper.js

@@ -70,6 +70,7 @@ App.statusMapper = App.QuickDataMapper.create({
       var currentHostStatuses = {};
       var previousHostStatuses = App.cache['previousHostStatuses'];
       var previousComponentStatuses = App.cache['previousComponentStatuses'];
+      var hostComponentsOnService = {};
 
       json.items.forEach(function (host) {
         //update hosts, which have status changed
@@ -81,6 +82,8 @@ App.statusMapper = App.QuickDataMapper.create({
         host.host_components.forEach(function (host_component) {
           host_component.id = host_component.HostRoles.component_name + "_" + host_component.HostRoles.host_name;
           var existedComponent = previousComponentStatuses[host_component.id];
+          var service = componentServiceMap[host_component.HostRoles.component_name];
+
           if (existedComponent) {
             //update host-components, which have status changed
             if (existedComponent !== host_component.HostRoles.state) {
@@ -92,11 +95,30 @@ App.statusMapper = App.QuickDataMapper.create({
               component_name: host_component.HostRoles.component_name,
               work_status: host_component.HostRoles.state,
               host_id: host.Hosts.host_name,
-              service_id: componentServiceMap[host_component.HostRoles.component_name]
+              service_id: service
             });
           }
-          hostComponentsOnHost.push(host_component.id);
           currentComponentStatuses[host_component.id] = host_component.HostRoles.state;
+
+          //host-components to host relations
+          hostComponentsOnHost.push(host_component.id);
+          //host-component to service relations
+          if (!hostComponentsOnService[service]) {
+            hostComponentsOnService[service] = {
+              host_components: [],
+              running_host_components: [],
+              unknown_host_components: []
+            };
+          }
+          if (host_component.HostRoles.state === App.HostComponentStatus.started) {
+            hostComponentsOnService[service].running_host_components.push(host_component.id);
+          }
+          if (host_component.HostRoles.state === App.HostComponentStatus.unknown) {
+            hostComponentsOnService[service].unknown_host_components.push(host_component.id);
+          }
+          hostComponentsOnService[service].host_components.push(host_component.id);
+
+
         }, this);
         /**
          * updating relation between Host and his host-components
@@ -131,6 +153,7 @@ App.statusMapper = App.QuickDataMapper.create({
 
       App.cache['previousHostStatuses'] = currentHostStatuses;
       App.cache['previousComponentStatuses'] = currentComponentStatuses;
+      App.cache['hostComponentsOnService'] = hostComponentsOnService;
 
       hosts.forEach(function (host) {
         var status = hostStatuses[host.get('id')];

ambari-web/app/models/service.js

@@ -30,7 +30,8 @@ App.Service = DS.Model.extend({
   quickLinks: DS.hasMany('App.QuickLinks'),
   hostComponents: DS.hasMany('App.HostComponent'),
   serviceConfigsTemplate: App.config.get('preDefinedServiceConfigs'),
-  runningHostComponents: null,
+  runningHostComponents: DS.hasMany('App.HostComponent'),
+  unknownHostComponents: DS.hasMany('App.HostComponent'),
 
   // Instead of making healthStatus a computed property that listens on hostComponents.@each.workStatus,
   // we are creating a separate observer _updateHealthStatus.  This is so that healthStatus is updated

ambari-web/app/models/service/hbase.js

@@ -20,7 +20,9 @@ var App = require('app');
 App.HBaseService = App.Service.extend({
   version: DS.attr('string'),
   master: DS.belongsTo('App.Host'),
-  regionServers: DS.hasMany('App.Host'),
+  regionServers: function(){
+    return this.get('hostComponents').filterProperty('componentName', 'HBASE_REGIONSERVER').mapProperty('host');
+  }.property('hostComponents.length'),
   masterStartTime: DS.attr('number'),
   masterActiveTime: DS.attr('number'),
   averageLoad: DS.attr('number'),

ambari-web/app/models/service/hdfs.js

@@ -24,7 +24,9 @@ App.HDFSService = App.Service.extend({
   activeNameNode: DS.belongsTo('App.Host'),
   standbyNameNode: DS.belongsTo('App.Host'),
   standbyNameNode2: DS.belongsTo('App.Host'),
-  dataNodes: DS.hasMany('App.Host'),
+  dataNodes: function(){
+    return this.get('hostComponents').filterProperty('componentName', 'DATANODE').mapProperty('host');
+  }.property('hostComponents.length'),
   journalNodes: DS.hasMany('App.Host'),
   nameNodeStartTime: DS.attr('number'),
   jvmMemoryHeapUsed: DS.attr('number'),

ambari-web/app/models/service/mapreduce.js

@@ -20,7 +20,9 @@ var App = require('app');
 App.MapReduceService = App.Service.extend({
   version: DS.attr('string'),
   jobTracker: DS.belongsTo('App.Host'),
-  taskTrackers: DS.hasMany('App.Host'),
+  taskTrackers: function () {
+    return this.get('hostComponents').filterProperty('componentName', 'TASKTRACKER').mapProperty('host');
+  }.property('hostComponents.length'),
   jobTrackerStartTime: DS.attr('number'),
   jobTrackerHeapUsed: DS.attr('number'),
   jobTrackerHeapMax: DS.attr('number'),

ambari-web/app/models/service/mapreduce2.js

@@ -20,7 +20,9 @@ var App = require('app');
 App.MapReduce2Service = App.Service.extend({
   version: DS.attr('string'),
   jobHistoryServer: DS.belongsTo('App.Host'),
-  mapReduce2Clients: DS.hasMany('App.Host')
+  mapReduce2Clients: function(){
+    return this.get('hostComponents').filterProperty('componentName', 'MAPREDUCE2_CLIENT').mapProperty('host');
+  }.property('hostComponents.length')
 });
 
 App.MapReduce2Service.FIXTURES = [];

ambari-web/app/models/service/yarn.js

@@ -21,7 +21,9 @@ var objectUtils = require('utils/object_utils');
 App.YARNService = App.Service.extend({
   version: DS.attr('string'),
   resourceManagerNode: DS.belongsTo('App.Host'),
-  nodeManagerNodes: DS.hasMany('App.Host'),
+  nodeManagerNodes: function(){
+    return this.get('hostComponents').filterProperty('componentName', 'NODEMANAGER').mapProperty('host');
+  }.property('hostComponents.length'),
   nodeManagerLiveNodes: DS.hasMany('App.Host'),
   nodeManagersCountActive: DS.attr('number'),
   nodeManagersCountUnhealthy: DS.attr('number'),
@@ -36,7 +38,9 @@ App.YARNService = App.Service.extend({
   appsCompleted: DS.attr('number'),
   appsKilled: DS.attr('number'),
   appsFailed: DS.attr('number'),
-  yarnClientNodes: DS.hasMany('App.Host'),
+  yarnClientNodes: function(){
+    return this.get('hostComponents').filterProperty('componentName', 'YARN_CLIENT').mapProperty('host');
+  }.property('hostComponents.length'),
   resourceManagerStartTime: DS.attr('number'),
   jvmMemoryHeapUsed: DS.attr('number'),
   jvmMemoryHeapCommitted: DS.attr('number'),
@@ -102,7 +106,7 @@ App.YARNService = App.Service.extend({
     var decomCount = this.get('nodeManagersCountDecommissioned');
     var nonLostHostsCount = activeCount + rebootedCount + decomCount + unhealthyCount;
     return totalCount >= nonLostHostsCount ? totalCount - nonLostHostsCount : 0;
-  }.property('nodeManagerNodes', 'nodeManagersCountActive', 'nodeManagersCountRebooted', 'nodeManagersCountUnhealthy', 'nodeManagersCountDecommissioned'),
+  }.property('nodeManagerNodes', 'nodeManagersCountActive', 'nodeManagersCountRebooted', 'nodeManagersCountUnhealthy', 'nodeManagersCountDecommissioned')
 });
 
 App.YARNService.FIXTURES = [];

ambari-web/app/views/main/dashboard/service/hdfs.js

@@ -55,9 +55,9 @@ App.MainDashboardServiceHdfsView = App.MainDashboardServiceView.extend({
   }.property('service.hostComponents.@each'),
 
   dataNodeHostText: function () {
-    if (this.get("service.dataNodes").content.length == 0) {
+    if (this.get("service.dataNodes").length == 0) {
       return '';
-    } else if (this.get("service.dataNodes").content.length > 1) {
+    } else if (this.get("service.dataNodes").length > 1) {
       return Em.I18n.t('services.service.summary.viewHosts');
     } else {
       return Em.I18n.t('services.service.summary.viewHost');