AMBARI-9418 UI changes to resolve discrepancies between what the stack vs Ambari reports as "live" for slaves such as DataNodes and NodeManagers. (ababiichuk)

ambari-web/app/controllers/global/update_controller.js

@@ -377,6 +377,8 @@ App.UpdateController = Em.Controller.extend({
         'host_components/metrics/rpc/RpcQueueTime_avg_time,' +
         'host_components/metrics/dfs/FSNamesystem/*,' +
         'host_components/metrics/dfs/namenode/Version,' +
+        'host_components/metrics/dfs/namenode/LiveNodes,' +
+        'host_components/metrics/dfs/namenode/DeadNodes,' +
         'host_components/metrics/dfs/namenode/DecomNodes,' +
         'host_components/metrics/dfs/namenode/TotalFiles,' +
         'host_components/metrics/dfs/namenode/UpgradeFinalized,' +
@@ -406,6 +408,7 @@ App.UpdateController = Em.Controller.extend({
         "host_components/processes/HostComponentProcess",
       'YARN': "host_components/metrics/yarn/Queue," +
         "ServiceComponentInfo/rm_metrics/cluster/activeNMcount," +
+        "ServiceComponentInfo/rm_metrics/cluster/lostNMcount," +
         "ServiceComponentInfo/rm_metrics/cluster/unhealthyNMcount," +
         "ServiceComponentInfo/rm_metrics/cluster/rebootedNMcount," +
         "ServiceComponentInfo/rm_metrics/cluster/decommissionedNMcount",

ambari-web/app/mappers/service_metrics_mapper.js

@@ -44,9 +44,12 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
     standby_name_node_id: 'standby_name_node_id',
     standby_name_node2_id: 'standby_name_node2_id',
     journal_nodes: 'journal_nodes',
+    metrics_not_available: 'metrics_not_available',
     name_node_start_time: 'nameNodeComponent.host_components[0].metrics.runtime.StartTime',
     jvm_memory_heap_used: 'nameNodeComponent.host_components[0].metrics.jvm.HeapMemoryUsed',
     jvm_memory_heap_max: 'nameNodeComponent.host_components[0].metrics.jvm.HeapMemoryMax',
+    live_data_nodes: 'live_data_nodes',
+    dead_data_nodes: 'dead_data_nodes',
     decommission_data_nodes: 'decommission_data_nodes',
     capacity_used: 'nameNodeComponent.host_components[0].metrics.dfs.FSNamesystem.CapacityUsed',
     capacity_total: 'nameNodeComponent.host_components[0].metrics.dfs.FSNamesystem.CapacityTotal',
@@ -79,7 +82,7 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
     apps_killed: 'resourceManagerComponent.host_components[0].metrics.yarn.Queue.root.AppsKilled',
     apps_failed: 'resourceManagerComponent.host_components[0].metrics.yarn.Queue.root.AppsFailed',
     node_managers_count_active: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.activeNMcount',
-    //node_managers_count_lost: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.lostNMcount',
+    node_managers_count_lost: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.lostNMcount',
     node_managers_count_unhealthy: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.unhealthyNMcount',
     node_managers_count_rebooted: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.rebootedNMcount',
     node_managers_count_decommissioned: 'resourceManagerComponent.ServiceComponentInfo.rm_metrics.cluster.decommissionedNMcount',
@@ -423,12 +426,25 @@ App.serviceMetricsMapper = App.QuickDataMapper.create({
         finalConfig = jQuery.extend(finalConfig, hdfsConfig);
         // Get the live, dead & decommission nodes from string json
         if (component.host_components[0].metrics && component.host_components[0].metrics.dfs && component.host_components[0].metrics.dfs.namenode) {
+          item.metrics_not_available = false;
           var decommissionNodesJson = App.parseJSON(component.host_components[0].metrics.dfs.namenode.DecomNodes);
+          var deadNodesJson = App.parseJSON(component.host_components[0].metrics.dfs.namenode.DeadNodes);
+          var liveNodesJson = App.parseJSON(component.host_components[0].metrics.dfs.namenode.LiveNodes);
+        } else {
+          item.metrics_not_available = true;
         }
         item.decommission_data_nodes = [];
+        item.dead_data_nodes = [];
+        item.live_data_nodes = [];
         for (var host in decommissionNodesJson) {
           item.decommission_data_nodes.push('DATANODE'+ '_' + host);
         }
+        for (var host in deadNodesJson) {
+          item.dead_data_nodes.push('DATANODE'+ '_' + host);
+        }
+        for (var host in liveNodesJson) {
+          item.live_data_nodes.push('DATANODE'+ '_' + host);
+        }
       }
       if (component.ServiceComponentInfo && component.ServiceComponentInfo.component_name == "JOURNALNODE") {
         if (!item.journal_nodes) {

ambari-web/app/messages.js

@@ -134,6 +134,7 @@ Em.I18n.translations = {
   'common.oss':'OSs',
   'common.memory':'Memory',
   'common.maximum':'Maximum',
+  'common.started':'Started',
   'common.start':'Start',
   'common.stop':'Stop',
   'common.pause':'Pause',
@@ -2261,6 +2262,19 @@ Em.I18n.translations = {
   'dashboard.services.hdfs.nodes.heapUsed':'{0} / {1} ({2}% used)',
   'dashboard.services.hdfs.chart.label':'Capacity (Used/Total)',
   'dashboard.services.hdfs.blockErrors':'{0} corrupt / {1} missing / {2} under replicated',
+  'dashboard.services.hdfs.datanode.status.tooltip.live': 'This is the number of DataNodes that are live as reported from ' +
+    'the NameNode. Even if a DataNode process is up, NameNode might see the status as dead ' +
+    'if the DataNode is not communicating with the NameNode as expected. This can be due situations ' +
+    'such as a network issue or a hanging DataNode process due to excessive garbage collection.',
+  'dashboard.services.hdfs.datanode.status.tooltip.dead': 'This is the number of DataNodes that are dead as reported from ' +
+  'the NameNode. Even if a DataNode process is up, NameNode might see the status as dead ' +
+  'if the DataNode is not communicating with the NameNode as expected. This can be due situations ' +
+  'such as a network issue or a hanging DataNode process due to excessive garbage collection.',
+  'dashboard.services.hdfs.datanode.status.tooltip.decommission': 'This is the number of DataNodes that are currently ' +
+  'Decommissioning as reported from the NameNode. If there are not enough other DataNodes ' +
+  'in the cluster to create the configured number of block replicas based on the dfs.replication ' +
+  'property (typically 3), a DataNode can get stuck in decommissioning state until ' +
+  'more DataNodes become available to the NameNode.',
 
   'dashboard.services.yarn.summary':'{0} of {1} nodes live',
   'dashboard.services.yarn.resourceManager':'ResourceManager',

ambari-web/app/models/service/hdfs.js

@@ -38,6 +38,8 @@ App.HDFSService = App.Service.extend({
   jvmMemoryHeapUsed: DS.attr('number'),
   jvmMemoryHeapMax: DS.attr('number'),
   decommissionDataNodes: DS.hasMany('App.HostComponent'),
+  liveDataNodes: DS.hasMany('App.HostComponent'),
+  deadDataNodes: DS.hasMany('App.HostComponent'),
   capacityUsed: DS.attr('number'),
   capacityTotal: DS.attr('number'),
   capacityRemaining: DS.attr('number'),
@@ -48,7 +50,8 @@ App.HDFSService = App.Service.extend({
   dfsTotalFiles: DS.attr('number'),
   upgradeStatus: DS.attr('boolean'),
   safeModeStatus: DS.attr('string'),
-  nameNodeRpc: DS.attr('number')
+  nameNodeRpc: DS.attr('number'),
+  metricsNotAvailable: DS.attr('boolean')
 });
 
 App.HDFSService.FIXTURES = [];

ambari-web/app/models/service/yarn.js

@@ -30,6 +30,7 @@ App.YARNService = App.Service.extend({
   nodeManagersInstalled: DS.attr('number'),
   nodeManagersTotal: DS.attr('number'),
   nodeManagersCountActive: DS.attr('number'),
+  nodeManagersCountLost: DS.attr('number'),
   nodeManagersCountUnhealthy: DS.attr('number'),
   nodeManagersCountRebooted: DS.attr('number'),
   nodeManagersCountDecommissioned: DS.attr('number'),
@@ -100,12 +101,12 @@ App.YARNService = App.Service.extend({
     }
     this.set('allQueueNames', allQueueNames);
     this.set('childQueueNames', childQueueNames);
-  }.observes('queue'),
+  }.observes('queue')
   /**
    * ResourceManager's lost count is not accurate once RM is rebooted. Since
    * Ambari knows the total number of nodes and the counts of nodes in other
    * states, we calculate the lost count.
-   */
+   *
   nodeManagersCountLost: function () {
     var totalCount = this.get('nodeManagersTotal');
     var activeCount = this.get('nodeManagersCountActive');
@@ -114,7 +115,7 @@ App.YARNService = App.Service.extend({
     var decomCount = this.get('nodeManagersCountDecommissioned');
     var nonLostHostsCount = activeCount + rebootedCount + decomCount + unhealthyCount;
     return totalCount >= nonLostHostsCount ? totalCount - nonLostHostsCount : 0;
-  }.property('nodeManagersTotal', 'nodeManagersCountActive', 'nodeManagersCountRebooted', 'nodeManagersCountUnhealthy', 'nodeManagersCountDecommissioned')
+  }.property('nodeManagersTotal', 'nodeManagersCountActive', 'nodeManagersCountRebooted', 'nodeManagersCountUnhealthy', 'nodeManagersCountDecommissioned')*/
 });
 
 App.YARNService.FIXTURES = [];

ambari-web/app/templates/main/service/services/hdfs.hbs

@@ -26,7 +26,16 @@
         {{view.liveComponents}}/{{view.totalComponents}}
       {{/view}}
     </span>
-      {{t services.service.summary.DataNodesLive}}
+      {{t common.started}}
+  </td>
+</tr>
+<!-- Data Node Counts -->
+<tr>
+  <td>{{t dashboard.services.hdfs.datanodecounts}}</td>
+  <td>
+    <span {{translateAttr data-original-title="dashboard.services.hdfs.datanode.status.tooltip.live" }} rel="tooltip">{{view.service.liveDataNodes.length}} {{t dashboard.services.hdfs.nodes.live}} </span> /
+    <span {{translateAttr data-original-title="dashboard.services.hdfs.datanode.status.tooltip.dead" }} rel="tooltip">{{view.service.deadDataNodes.length}} {{t dashboard.services.hdfs.nodes.dead}} </span> /
+    <span {{translateAttr data-original-title="dashboard.services.hdfs.datanode.status.tooltip.decommission" }} rel="tooltip">{{view.service.decommissionDataNodes.length}} {{t dashboard.services.hdfs.nodes.decom}}</span>
   </td>
 </tr>
 <!-- JournalNodes -->
@@ -54,15 +63,6 @@
   <td>{{t dashboard.services.hdfs.nodes.heap}}</td>
   <td>{{view.nodeHeap}}</td>
 </tr>
-<!-- Data Node Counts -->
-<tr>
-  <td>{{t dashboard.services.hdfs.datanodecounts}}</td>
-  <td>
-    {{view.dataNodesLive}} {{t dashboard.services.hdfs.nodes.live}} /
-    {{view.dataNodesDead}} {{t dashboard.services.hdfs.nodes.dead}} /
-    {{view.service.decommissionDataNodes.length}} {{t dashboard.services.hdfs.nodes.decom}}
-  </td>
-</tr>
 <!-- HDFS Capacity (Disk Usage)-->
 <tr>
   <td>{{t dashboard.services.hdfs.capacity.dfsUsed}}</td>

ambari-web/app/templates/main/service/services/yarn.hbs

@@ -25,9 +25,14 @@
       {{#view App.ComponentLiveTextView liveComponentsBinding="view.service.nodeManagersStarted" totalComponentsBinding="view.service.nodeManagersTotal"}}
         {{view.liveComponents}}/{{view.totalComponents}}
       {{/view}}
-    </span>{{t services.service.summary.nodeManagersLive}}
+    </span>{{t common.started}}
   </td>
 </tr>
+<!-- NodeManagers status -->
+<tr>
+  <td>{{t dashboard.services.yarn.nodeManagers.status}}</td>
+  <td>{{view.nodeManagersStatus}}</td>
+</tr>
 <!-- YARN Clients -->
 <tr>
   <td class="summary-label"><a {{action filterHosts view.yarnClientComponent}}
@@ -48,11 +53,6 @@
   <td>{{t dashboard.services.resourceManager.nodes.heap}}</td>
   <td>{{view.nodeHeap}}</td>
 </tr>
-<!-- NodeManagers status -->
-<tr>
-  <td>{{t dashboard.services.yarn.nodeManagers.status}}</td>
-  <td>{{view.nodeManagersStatus}}</td>
-</tr>
 <!-- Containers -->
 <tr>
   <td>{{t dashboard.services.yarn.containers}}</td>

ambari-web/app/views/main/dashboard/widgets/datanode_live.js

@@ -32,9 +32,9 @@ App.DataNodeUpView = App.TextDashboardWidgetView.extend({
     var result = [];
     result.pushObject(this.get('dataNodesLive') + ' ' + Em.I18n.t('dashboard.services.hdfs.nodes.live'));
     result.pushObject(this.get('dataNodesDead') + ' ' + Em.I18n.t('dashboard.services.hdfs.nodes.dead'));
-    result.pushObject(this.get('model.decommissionDataNodes.length')+ ' ' + Em.I18n.t('dashboard.services.hdfs.nodes.decom'));
+    result.pushObject(this.get('dataNodesDecom')+ ' ' + Em.I18n.t('dashboard.services.hdfs.nodes.decom'));
     return result;
-  }.property('dataNodesLive', 'dataNodesDead', 'model.decommissionDataNodes.length'),
+  }.property('dataNodesLive', 'dataNodesDead', 'dataNodesDecom'),
   hiddenInfoClass: "hidden-info-three-line",
 
   thresh1: 40,
@@ -42,22 +42,25 @@ App.DataNodeUpView = App.TextDashboardWidgetView.extend({
   maxValue: 100,
 
   dataNodesLive: function () {
-    return this.get('model.dataNodesStarted');
-  }.property('model.dataNodesStarted'),
+    return this.get('model.metricsNotAvailable') ? Em.I18n.t('services.service.summary.notAvailable') : this.get('model.liveDataNodes.length');
+  }.property('model.liveDataNodes.length'),
   dataNodesDead: function () {
-    return this.get('model.dataNodesInstalled');
-  }.property('model.dataNodesInstalled'),
+    return this.get('model.metricsNotAvailable') ? Em.I18n.t('services.service.summary.notAvailable') : this.get('model.deadDataNodes.length');
+  }.property('model.deadDataNodes.length'),
+  dataNodesDecom: function () {
+    return this.get('model.metricsNotAvailable') ? Em.I18n.t('services.service.summary.notAvailable') : this.get('model.decommissionDataNodes.length');
+  }.property('model.decommissionDataNodes.length'),
 
   data: function () {
-    if ( !this.get('model.dataNodesTotal')) {
-      return -1;
+    if (this.get('model.metricsNotAvailable')) {
+      return null;
     } else {
       return ((this.get('dataNodesLive') / this.get('model.dataNodesTotal')).toFixed(2)) * 100;
     }
   }.property('model.dataNodesTotal', 'dataNodesLive'),
 
   content: function () {
-    return this.get('dataNodesLive') + "/" + this.get('model.dataNodesTotal');
+    return this.get('model.metricsNotAvailable') ? Em.I18n.t('services.service.summary.notAvailable') : this.get('dataNodesLive') + "/" + this.get('model.dataNodesTotal');
   }.property('model.dataNodesTotal', 'dataNodesLive'),
 
   editWidget: function (event) {

ambari-web/app/views/main/dashboard/widgets/node_managers_live.js

@@ -47,21 +47,21 @@ App.NodeManagersLiveView = App.TextDashboardWidgetView.extend({
   maxValue: 100,
 
   nodeManagersLive: function () {
-    return this.get('model.nodeManagersStarted');
-  }.property('model.nodeManagersStarted'),
+    return this.get('model.nodeManagersCountActive');
+  }.property('model.nodeManagersCountActive'),
 
   data: function () {
     var nodeManagers = this.get('model.nodeManagersTotal');
     var nodeManagersLive = this.get('nodeManagersLive');
-    if (!nodeManagers) {
-      return -1;
+    if (nodeManagersLive == null) {
+      return null;
     } else {
       return (nodeManagersLive / nodeManagers).toFixed(2) * 100;
     }
   }.property('model.nodeManagersTotal', 'nodeManagersLive'),
 
   content: function () {
-    return this.get('nodeManagersLive') + '/' + this.get('model.nodeManagersTotal');
+    return this.get('nodeManagersLive') == null ? Em.I18n.t('services.service.summary.notAvailable') : this.get('nodeManagersLive') + '/' + this.get('model.nodeManagersTotal');
   }.property('model.nodeManagersTotal', 'nodeManagersLive'),
 
   editWidget: function (event) {

ambari-web/app/views/main/service/services/hdfs.js

@@ -60,6 +60,9 @@ App.MainDashboardServiceHdfsView = App.MainDashboardServiceView.extend({
     }.property('parentView.service.hostComponents.length')
   }),
 
+  didInsertElement: function() {
+    App.tooltip($("[rel='tooltip']"));
+  },
   dataNodesLive: function () {
     return this.get('service.dataNodesStarted');
   }.property('service.dataNodesStarted'),

ambari-web/test/controllers/global/update_controller_test.js

@@ -149,6 +149,7 @@ describe('App.UpdateController', function () {
         ],
         result: ["host_components/metrics/yarn/Queue," +
           "ServiceComponentInfo/rm_metrics/cluster/activeNMcount," +
+          "ServiceComponentInfo/rm_metrics/cluster/lostNMcount," +
           "ServiceComponentInfo/rm_metrics/cluster/unhealthyNMcount," +
           "ServiceComponentInfo/rm_metrics/cluster/rebootedNMcount," +
           "ServiceComponentInfo/rm_metrics/cluster/decommissionedNMcount"]

ambari-web/test/models/service/yarn_test.js

@@ -152,7 +152,7 @@ describe('App.YARNService', function () {
       expect(yarnService.get('childQueueNames')).to.eql(['root/default']);
     });
   });
-
+/*
   describe('#nodeManagersCountLost', function () {
     nodeCountCases.forEach(function (item) {
       it('should be ' + item.nodeManagersCountLost, function () {
@@ -164,5 +164,5 @@ describe('App.YARNService', function () {
       });
     });
   });
-
+*/
 });