AMBARI-5078. Job Tez DAG should show spilled records per vertex. (xiwang)

ambari-web/app/messages.js

@@ -1954,6 +1954,7 @@ Em.I18n.translations = {
   'jobs.hive.tez.tasks':'Tez Tasks',
   'jobs.hive.tez.hdfs':'HDFS',
   'jobs.hive.tez.localFiles':'Local Files',
+  'jobs.hive.tez.spilledRecords':'Spilled Records',
   'jobs.hive.tez.records':'Records',
   'jobs.hive.tez.reads':'{0} reads',
   'jobs.hive.tez.writes':'{0} writes',
@@ -1971,6 +1972,7 @@ Em.I18n.translations = {
   'jobs.hive.tez.metric.recordsRead':'Records Read',
   'jobs.hive.tez.metric.recordsWrite':'Records Written',
   'jobs.hive.tez.metric.tezTasks':'Tez Tasks',
+  'jobs.hive.tez.metric.spilledRecords':'Spilled Records',
   'jobs.hive.tez.edge.':'Unknown',
   'jobs.hive.tez.edge.contains':'Contains',
   'jobs.hive.tez.edge.broadcast':'Broadcast',

ambari-web/app/models/jobs/tez_dag.js

@@ -118,6 +118,11 @@ App.TezDagVertex = DS.Model.extend({
   fileReadOps : DS.attr('number'),
   fileWriteOps : DS.attr('number'),
 
+  /**
+   * Spilled records
+   */
+  spilledRecords : DS.attr('number'),
+
   /**
    * HDFS usage metrics for this vertex
    */

ambari-web/app/templates/main/jobs/hive_job_details.hbs

@@ -161,6 +161,10 @@
                       <td>{{view.selectedVertexIODisplay.file.read.ops}} / {{view.selectedVertexIODisplay.file.read.bytes}}</td>
                       <td>{{view.selectedVertexIODisplay.file.write.ops}} / {{view.selectedVertexIODisplay.file.write.bytes}}</td>
                     </tr>
+                    <tr>
+                        <td>{{t jobs.hive.tez.spilledRecords}}</td>
+                        <td>{{view.selectedVertex.spilledRecords}}</td>
+                    </tr>
                     {{#if view.selectedVertexIODisplay.records.read}}
                       <tr>
                         <td>{{t jobs.hive.tez.records}}</td>

ambari-web/app/utils/jobs.js

@@ -197,6 +197,11 @@ module.exports = {
                     'HDFS_WRITE_OPS' : 'hdfsWriteOps'
                   };
                   break;
+                case 'org.apache.tez.common.counters.TaskCounter':
+                  cNameToPropetyMap = {
+                     'SPILLED_RECORDS' : 'spilledRecords'
+                    };
+                  break;
                 case 'HIVE':
                   cNameToPropetyMap = {
                     'RECORDS_READ' : 'recordReadCount',

ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js

@@ -212,14 +212,16 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       output : Number.MAX_VALUE,
       recordsRead : Number.MAX_VALUE,
       recordsWrite : Number.MAX_VALUE,
-      tezTasks : Number.MAX_VALUE
+      tezTasks : Number.MAX_VALUE,
+      spilledRecords : Number.MAX_VALUE
     };
     dagVisualModel.maxMetrics = {
       input : 0,
       output : 0,
       recordsRead : 0,
       recordsWrite : 0,
-      tezTasks : 0
+      tezTasks : 0,
+      spilledRecords : 0
     };
     if (dagVisualModel.nodes) {
       dagVisualModel.nodes.forEach(function(node) {
@@ -230,6 +232,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
           node.metrics['recordsRead'] = vertex.get('recordReadCount');
           node.metrics['recordsWrite'] = vertex.get('recordWriteCount');
           node.metrics['tezTasks'] = vertex.get('tasksCount');
+          node.metrics['spilledRecords'] = vertex.get('spilledRecords');
           node.state = vertex.get('state');
           // Min metrics
           dagVisualModel.minMetrics.input = Math.min(dagVisualModel.minMetrics.input, node.metrics.input);
@@ -237,12 +240,14 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
           dagVisualModel.minMetrics.recordsRead = Math.min(dagVisualModel.minMetrics.recordsRead, node.metrics.recordsRead);
           dagVisualModel.minMetrics.recordsWrite = Math.min(dagVisualModel.minMetrics.recordsWrite, node.metrics.recordsWrite);
           dagVisualModel.minMetrics.tezTasks = Math.min(dagVisualModel.minMetrics.tezTasks, node.metrics.tezTasks);
+          dagVisualModel.minMetrics.spilledRecords = Math.min(dagVisualModel.minMetrics.spilledRecords, node.metrics.spilledRecords);
           // Max metrics
           dagVisualModel.maxMetrics.input = Math.max(dagVisualModel.maxMetrics.input, node.metrics.input);
           dagVisualModel.maxMetrics.output = Math.max(dagVisualModel.maxMetrics.output, node.metrics.output);
           dagVisualModel.maxMetrics.recordsRead = Math.max(dagVisualModel.maxMetrics.recordsRead, node.metrics.recordsRead);
           dagVisualModel.maxMetrics.recordsWrite = Math.max(dagVisualModel.maxMetrics.recordsWrite, node.metrics.recordsWrite);
           dagVisualModel.maxMetrics.tezTasks = Math.max(dagVisualModel.maxMetrics.tezTasks, node.metrics.tezTasks);
+          dagVisualModel.maxMetrics.spilledRecords = Math.max(dagVisualModel.maxMetrics.spilledRecords, node.metrics.spilledRecords);
         }
       });
     }
@@ -250,7 +255,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
   }.observes('content.tezDag.vertices.@each.fileReadBytes', 'content.tezDag.vertices.@each.fileWriteBytes', 
       'content.tezDag.vertices.@each.hdfsReadBytes', 'content.tezDag.vertices.@each.hdfsWriteBytes', 
       'content.tezDag.vertices.@each.recordReadCount', 'content.tezDag.vertices.@each.recordWriteCount',
-      'content.tezDag.vertices.@each.state'),
+      'content.tezDag.vertices.@each.state', 'content.tezDag.vertices.@each.spilledRecords'),
 
   /**
    * Determines layout and creates Tez graph. In the process it populates the

ambari-web/app/views/main/jobs/hive_job_details_view.js

@@ -51,7 +51,7 @@ App.MainHiveJobDetailsView = Em.View.extend({
 
   summaryMetricType: 'input',
   summaryMetricTypesDisplay : [ Em.I18n.t('jobs.hive.tez.metric.input'), Em.I18n.t('jobs.hive.tez.metric.output'), Em.I18n.t('jobs.hive.tez.metric.recordsRead'),
-                                Em.I18n.t('jobs.hive.tez.metric.recordsWrite'), Em.I18n.t('jobs.hive.tez.metric.tezTasks') ],
+                                Em.I18n.t('jobs.hive.tez.metric.recordsWrite'), Em.I18n.t('jobs.hive.tez.metric.tezTasks'), Em.I18n.t('jobs.hive.tez.metric.spilledRecords') ],
   summaryMetricTypeDisplay: function(){
     return Em.I18n.t('jobs.hive.tez.metric.'+this.get('summaryMetricType'));
   }.property('summaryMetricType'),
@@ -121,6 +121,9 @@ App.MainHiveJobDetailsView = Em.View.extend({
     case Em.I18n.t('jobs.hive.tez.metric.tezTasks'):
       summaryType = 'tezTasks';
       break;
+    case Em.I18n.t('jobs.hive.tez.metric.spilledRecords'):
+      summaryType = 'spilledRecords';
+      break;
     default:
       break;
     }