AMBARI-4751. ATS api response for Tez vertices missing metrics. (srimanth)

ambari-web/app/controllers/main/jobs_controller.js

@@ -192,7 +192,7 @@ App.MainJobsController = Em.ArrayController.extend({
      * @return {String}
      */
     createJobsFiltersLink: function() {
-      var link = "?fields=events,primaryfilters";
+      var link = "?fields=events,primaryfilters,otherinfo";
 
       if(this.get("id") !== "") {
         link = "/" + this.get("id") + link;

ambari-web/app/mappers/jobs/hive_job_mapper.js

@@ -57,7 +57,6 @@ App.hiveJobMapper = App.QuickDataMapper.create({
       hiveJob.startTime = json.starttime;
       hiveJob.endTime = json.endtime;
       json.otherinfo.query = $.parseJSON(json.otherinfo.query).query;
-      hiveJob.queryText = json.otherinfo.query.queryText;
       hiveJob.stages = [];
       var stagePlans = json.otherinfo.query.queryPlan["STAGE PLANS"];
       for ( var stage in stagePlans) {
@@ -163,11 +162,13 @@ App.hiveJobMapper = App.QuickDataMapper.create({
       }
       var hiveJobRecord = App.HiveJob.find(hiveJob.id);
       if (hiveJobRecord != null) {
-        hiveJobRecord.set('queryText', hiveJob.queryText);
         hiveJobRecord.set('stages', hiveJob.stages.sort(sortById));
         hiveJobRecord.set('startTime', hiveJob.startTime);
         hiveJobRecord.set('endTime', hiveJob.endTime);
-        hiveJobRecord.set('tezDag', App.TezDag.find(hiveJob.tezDag));
+        if (hiveJob.tezDag != null) {
+          // Some hive queries dont use Tez
+          hiveJobRecord.set('tezDag', App.TezDag.find(hiveJob.tezDag));
+        }
       }
     }
   },

ambari-web/app/mappers/jobs/hive_jobs_mapper.js

@@ -34,6 +34,15 @@ App.hiveJobsMapper = App.QuickDataMapper.create({
           name : entity.entity,
           user : entity.primaryfilters.user
         }
+        hiveJob.has_tez_dag = false;
+        hiveJob.query_text = '';
+        if (entity.otherinfo && entity.otherinfo.query) {
+          hiveJob.has_tez_dag = entity.otherinfo.query.match("\"Tez\".*\"DagName:\"");
+          var queryJson = $.parseJSON(entity.otherinfo.query);
+          if (queryJson && queryJson.query) {
+            hiveJob.query_text = queryJson.query.queryText;
+          }
+        }
         if (entity.events != null) {
           entity.events.forEach(function(event) {
             switch (event.eventtype) {
@@ -48,6 +57,12 @@ App.hiveJobsMapper = App.QuickDataMapper.create({
             }
           });
         }
+        if (!hiveJob.start_time && entity.starttime > 0) {
+          hiveJob.start_time = entity.starttime;
+        }
+        if (!hiveJob.end_time && entity.endtime > 0) {
+          hiveJob.end_time = entity.endtime;
+        }
         hiveJobs.push(hiveJob);
       });
       // Delete IDs not seen from server

ambari-web/app/models/jobs/hive_job.js

@@ -21,6 +21,7 @@ App.HiveJob = App.AbstractJob.extend({
   jobType : App.JobType.HIVE,
   queryText : DS.attr('string'),
   stages : DS.attr('array'),
+  hasTezDag: DS.attr('boolean'),
   tezDag : DS.belongsTo('App.TezDag')
 });
 

ambari-web/app/routes/main.js

@@ -149,7 +149,9 @@ module.exports = Em.Route.extend({
       }
     }),
     showJobDetails : function(router, event) {
-      router.transitionTo('jobDetails', event.context);
+      if (event.context && event.context.get('hasTezDag')) {
+        router.transitionTo('jobDetails', event.context);
+      }
     }
   }),
 

ambari-web/app/templates/main/jobs.hbs

@@ -56,7 +56,11 @@
           {{#each job in view.pageContent}}
             <tr>
               <td class="id">
-                <a title="{{unbound job.name}}" href="#" {{action "showJobDetails" job}}>{{unbound job.name}}</a>
+                {{#if job.hasTezDag}}
+                  <a rel="tooltip" class="job-link" title="{{unbound job.queryText}}" href="#" {{action "showJobDetails" job}}>{{unbound job.name}}</a>
+                {{else}}
+                  <span rel="tooltip" class="job-link" title="{{unbound job.queryText}}">{{unbound job.name}}</a>
+                {{/if}}
               </td>
               <td>
                 {{job.user}}

ambari-web/app/templates/main/jobs/hive_job_details.hbs

@@ -140,11 +140,14 @@
                       <td>{{view.selectedVertexIODisplay.file.read.ops}} / {{view.selectedVertexIODisplay.file.read.bytes}}</td>
                       <td>{{view.selectedVertexIODisplay.file.write.ops}} / {{view.selectedVertexIODisplay.file.write.bytes}}</td>
                     </tr>
+                    {{#if view.selectedVertexIODisplay.records.read}}
+                      <tr>
+                        <td>{{t jobs.hive.tez.records}}</td>
+                        <td>{{view.selectedVertexIODisplay.records.read}}</td>
+                        <td>{{view.selectedVertexIODisplay.records.write}}</td>
+                      </tr>
+                    {{/if}}
                     <tr>
-                      <td>{{t jobs.hive.tez.records}}</td>
-                      <td>{{view.selectedVertexIODisplay.records.read}}</td>
-                      <td>{{view.selectedVertexIODisplay.records.write}}</td>
-                    </tr>
                       <td>{{t jobs.hive.tez.operatorPlan}}</td>
                       <td></td>
                       <td></td>

ambari-web/app/utils/jobs.js

@@ -162,17 +162,41 @@ module.exports = {
             vertexRecord.set('endTime', data.otherinfo.endTime);
             vertexRecord.set('tasksCount', data.otherinfo.numTasks);
             vertexRecord.set('state', data.otherinfo.status);
-            // TODO Need additional vertex metrics
-            vertexRecord.set('fileReadBytes', 0);
-            vertexRecord.set('fileReadOps', 0);
-            vertexRecord.set('fileWriteOps', 0);
-            vertexRecord.set('fileWriteBytes', 0);
-            vertexRecord.set('hdfsReadOps', 0);
-            vertexRecord.set('hdfsReadBytes', 0);
-            vertexRecord.set('hdfsWriteOps', 0);
-            vertexRecord.set('hdfsWriteBytes', 0);
-            vertexRecord.set('recordReadCount', 0);
-            vertexRecord.set('recordWriteCount', 0);
+            if (data.otherinfo.counters && data.otherinfo.counters.counterGroups) {
+              data.otherinfo.counters.counterGroups.forEach(function(cGroup) {
+                var cNameToPropetyMap = {};
+                switch (cGroup.counterGroupName) {
+                case 'org.apache.tez.common.counters.FileSystemCounter':
+                  cNameToPropetyMap = {
+                    'FILE_BYTES_READ' : 'fileReadBytes',
+                    'FILE_BYTES_WRITTEN' : 'fileWriteBytes',
+                    'FILE_READ_OPS' : 'fileReadOps',
+                    'FILE_WRITE_OPS' : 'fileWriteOps',
+                    'HDFS_BYTES_READ' : 'hdfsReadBytes',
+                    'HDFS_BYTES_WRITTEN' : 'hdfsWriteBytes',
+                    'HDFS_READ_OPS' : 'hdfsReadOps',
+                    'HDFS_WRITE_OPS' : 'hdfsWriteOps'
+                  };
+                  break;
+                case 'HIVE':
+                  cNameToPropetyMap = {
+                    'RECORDS_READ' : 'recordReadCount',
+                    'RECORDS_WRITE' : 'recordWriteCount'
+                  };
+                  break;
+                default:
+                  break;
+                }
+                if (cGroup.counters) {
+                  cGroup.counters.forEach(function(counter) {
+                    var prop = cNameToPropetyMap[counter.counterName];
+                    if (prop != null) {
+                      vertexRecord.set(prop, counter.counterValue);
+                    }
+                  });
+                }
+              });
+            }
             successCallback();
           }
         }

ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js

@@ -375,7 +375,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
     });
     var link = svgLayer.selectAll(".link").data(dagVisualModel.links).enter().append("g").attr("class", "link").attr("marker-end", "url(#arrow)");
     link.append("path").attr("class", function(l) {
-      var classes = "link ";
+      var classes = "link svg-tooltip ";
       switch (l.edgeType) {
       case App.TezDagVertexType.BROADCAST:
         classes += "type-broadcast ";
@@ -387,7 +387,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
         break;
       }
       return classes;
-    }).attr("d", diagonal).append("title").text(function(l) {
+    }).attr("d", diagonal).attr("title", function(l) {
       var lower = l.edgeType ? l.edgeType.toLowerCase() : '';
       return Em.I18n.t("jobs.hive.tez.edge."+lower);
     });
@@ -412,7 +412,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
         var column = opIndex % 3;
         return "translate(" + (10 + column * 50) + "," + (37 + row * 20) + ")";
       }).attr("clip-path", "url(#operatorClipPath)");
-      opGroups.append("rect").attr("class", "operation").attr("width", "44").attr("height", "16").append("title").text(function(op) {
+      opGroups.append("rect").attr("class", "operation svg-tooltip ").attr("width", "44").attr("height", "16").attr("title", function(op) {
         return op;
       });
       opGroups.append("text").attr("x", "2").attr("dy", "1em").text(function(op) {
@@ -420,8 +420,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       })
     })
     var metricNodes = node.append("g").attr("class", "metric").attr("transform", "translate(92,7)");
-    metricNodes.append("rect").attr("width", 60).attr("height", 18).attr("rx", "3");
-    metricNodes.append("title").attr("class", "metric-title");
+    metricNodes.append("rect").attr("width", 60).attr("height", 18).attr("rx", "3").attr("class", "metric-title svg-tooltip");
     metricNodes.append("text").attr("class", "metric-text").attr("x", "2").attr("dy", "1em");
     node.append("text").attr("x", "1.9em").attr("dy", "1.5em").text(function(d) {
       return d.name;
@@ -431,6 +430,9 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       return "translate(" + d.x + "," + d.y + ")";
     });
     this.vertexMetricsUpdated();
+    $('.svg-tooltip').tooltip({
+      placement : 'left'
+    });
   },
 
   /**
@@ -470,7 +472,9 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       metricNodeTexts.text(function(node){
         return node.metricDisplay;
       });
-      metricNodeTitles.text(function(node){
+      metricNodeTitles.attr("title", function(node){
+        return node.metricType;
+      }).attr("data-original-title", function(node){
         return node.metricType;
       });
       nodeBackgrounds.attr("class", function(n) {

ambari-web/app/views/main/jobs/hive_job_details_view.js

@@ -169,8 +169,8 @@ App.MainHiveJobDetailsView = Em.View.extend({
         }
       },
       records : {
-        read : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordReadCount')),
-        write : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordWriteCount'))
+        read : v.get('recordReadCount') == null ? null : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordReadCount')),
+        write : v.get('recordWriteCount') == null ? null : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordWriteCount'))
       },
       started: v.get('startTime') ? dateUtils.dateFormat(v.get('startTime')) : '',
       ended: v.get('endTime') ? dateUtils.dateFormat(v.get('endTime')) : '',

ambari-web/app/views/main/jobs_view.js

@@ -99,6 +99,16 @@ App.MainJobsView = App.TableView.extend({
     return Em.I18n.t('jobs.filtered.jobs').format(this.get('content').get('length'), this.get('controller.totalOfJobs'));
   }.property('content.length', 'filteredContent.length', 'controller.totalOfJobs'),
 
+  pageContentObserver: function () {
+    Ember.run.later(this, function() {
+      $('.job-link').tooltip();
+    }, 1000);
+  }.observes('pageContent', 'pageContent.length', 'pageContent.@each.id'),
+
+  willDestroyElement : function() {
+    $('.job-link').tooltip('destroy');
+  },
+
   /**
    * Filter-field for Jobs ID.
    * Based on <code>filters</code> library