AMBARI-4493. Jobs: Implement the linear Tez Vertices table. (srimanth)

ambari-web/app/controllers.js

@@ -109,6 +109,8 @@ require('controllers/main/charts/heatmap_metrics/heatmap_metric_hbase_memstoresi
 require('controllers/main/charts/heatmap');
 require('controllers/main/apps_controller');
 require('controllers/main/jobs_controller');
+require('controllers/main/jobs/hive_job_details_controller');
+require('controllers/main/apps_controller');
 require('controllers/main/apps/item_controller');
 require('controllers/main/mirroring_controller');
 require('controllers/main/mirroring/edit_dataset_controller');

ambari-web/app/controllers/main/jobs/hive_job_details_controller.js

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+
+App.MainHiveJobDetailsController = Em.Controller.extend({
+  name : 'mainHiveJobDetailsController',
+  content : null,
+
+  /**
+   * path to page visited before
+   */
+  referer : '',
+  /**
+   * open dashboard page
+   */
+  routeHome : function() {
+    App.router.transitionTo('main.dashboard');
+  },
+
+  /**
+   * open jobs page
+   *
+   * @param event
+   */
+  routeToJobs : function() {
+    App.router.transitionTo('main.jobs.index');
+  }
+});

ambari-web/app/mappers/jobs/hive_job_mapper.js

@@ -62,7 +62,7 @@ App.hiveJobMapper = App.QuickDataMapper.create({
             var vertex = stageValue.Tez["Vertices:"][vertexName];
             var vertexObj = {
               id : dagName + "/" + vertexName,
-              name : vertexName,
+              name : vertexName
             };
             vertexIds.push(vertexObj.id);
             var operatorExtractor = function(obj) {
@@ -82,13 +82,13 @@ App.hiveJobMapper = App.QuickDataMapper.create({
               return ops;
             }
             if (vertex["Map Operator Tree:"] != null) {
-              vertexObj.isMap = true;
+              vertexObj.is_map = true;
               vertexObj.operations = operatorExtractor(vertex["Map Operator Tree:"]);
-              vertexObj.operationPlan = JSON.stringify(vertex["Map Operator Tree:"]);
+              vertexObj.operation_plan = JSON.stringify(vertex["Map Operator Tree:"], undefined, "  ");
             } else if (vertex["Reduce Operator Tree:"] != null) {
-              vertexObj.isMap = false;
+              vertexObj.is_map = false;
               vertexObj.operations = operatorExtractor(vertex["Reduce Operator Tree:"]);
-              vertexObj.operationPlan = JSON.stringify(vertex["Reduce Operator Tree:"]);
+              vertexObj.operation_plan = JSON.stringify(vertex["Reduce Operator Tree:"], undefined, "  ");
             }
             vertices.push(vertexObj);
           }
@@ -96,20 +96,25 @@ App.hiveJobMapper = App.QuickDataMapper.create({
           var edges = [];
           var edgeIds = [];
           for ( var childVertex in stageValue.Tez["Edges:"]) {
-            stageValue.Tez["Edges:"][childVertex].forEach(function(e) {
+            var childVertices = stageValue.Tez["Edges:"][childVertex];
+            if (!$.isArray(childVertices)) {
+              // Single edge given as object instead of array
+              childVertices = [ childVertices ];
+            }
+            childVertices.forEach(function(e) {
               var parentVertex = e.parent;
               var edgeObj = {
                 id : dagName + "/" + parentVertex + "-" + childVertex,
-                fromVertex : dagName + "/" + parentVertex,
-                toVertex : dagName + "/" + childVertex
+                from_vertex : dagName + "/" + parentVertex,
+                to_vertex : dagName + "/" + childVertex
               };
               edgeIds.push(edgeObj.id);
               switch (e.type) {
               case "BROADCAST_EDGE":
-                edgeObj.edgeType = App.TezDagVertexType.BROADCAST;
+                edgeObj.edge_type = App.TezDagVertexType.BROADCAST;
                 break;
               case "SIMPLE_EDGE":
-                edgeObj.edgeType = App.TezDagVertexType.SCATTER_GATHER;
+                edgeObj.edge_type = App.TezDagVertexType.SCATTER_GATHER;
                 break;
               default:
                 break;

ambari-web/app/messages.js

@@ -1829,6 +1829,15 @@ Em.I18n.translations = {
   'menu.item.jobs':'Jobs',
   'menu.item.admin':'Admin',
 
+  'jobs.hive.tez.tasks':'Tez Tasks',
+  'jobs.hive.tez.hdfs':'HDFS',
+  'jobs.hive.tez.localFiles':'Local Files',
+  'jobs.hive.tez.records':'Records',
+  'jobs.hive.tez.reads':'{0} reads',
+  'jobs.hive.tez.writes':'{0} writes',
+  'jobs.hive.tez.records.count':'{0} Records',
+  'jobs.hive.tez.operatorPlan':'Operator Plan',
+
   'number.validate.empty': 'cannot be empty',
   'number.validate.notValidNumber': 'not a valid number',
   'number.validate.lessThanMinumum': 'value less than {0}',

ambari-web/app/models/jobs/tez_dag.js

@@ -17,13 +17,14 @@
 
 var App = require('app');
 var dateUtils = require('utils/date');
+var numberUtils = require('utils/number_utils');
 
 App.TezDag = DS.Model.extend({
   id : DS.attr('string'),
   /**
    * When DAG is actually running on server, a unique ID is assigned.
    */
-  insanceId : DS.attr('string'),
+  instanceId : DS.attr('string'),
   name : DS.attr('string'),
   stage : DS.attr('string'),
   vertices : DS.hasMany('App.TezDagVertex'),
@@ -32,7 +33,7 @@ App.TezDag = DS.Model.extend({
 
 App.TezDagEdge = DS.Model.extend({
   id : DS.attr('string'),
-  insanceId : DS.attr('string'),
+  instanceId : DS.attr('string'),
   fromVertex : DS.belongsTo('App.TezDagVertex'),
   toVertex : DS.belongsTo('App.TezDagVertex'),
   /**
@@ -47,7 +48,7 @@ App.TezDagVertex = DS.Model.extend({
   /**
    * When DAG vertex is actually running on server, a unique ID is assigned.
    */
-  insanceId : DS.attr('string'),
+  instanceId : DS.attr('string'),
   name : DS.attr('string'),
 
   /**
@@ -88,8 +89,10 @@ App.TezDagVertex = DS.Model.extend({
    * Each Tez vertex can perform arbitrary application specific computations
    * inside. The application can provide a list of operations it has provided in
    * this vertex.
+   *
+   * Array of strings. [{string}]
    */
-  operations : DS.hasMany('string'),
+  operations : [],
 
   /**
    * Provides additional information about the 'operations' performed in this
@@ -122,7 +125,20 @@ App.TezDagVertex = DS.Model.extend({
    * Record metrics for this vertex
    */
   recordReadCount : DS.attr('number'),
-  recordWriteCount : DS.attr('number')
+  recordWriteCount : DS.attr('number'),
+
+  totalReadBytesDisplay : function() {
+    return numberUtils.bytesToSize(this.get('fileReadBytes') + this.get('hdfsReadBytes'));
+  }.property('fileReadBytes', 'hdfsReadBytes'),
+
+  totalWriteBytesDisplay : function() {
+    return numberUtils.bytesToSize(this.get('fileWriteBytes') + this.get('hdfsWriteBytes'));
+  }.property('fileWriteBytes', 'hdfsWriteBytes'),
+
+  durationDisplay : function() {
+    var duration = this.get('duration');
+    return dateUtils.timingFormat(duration, true);
+  }.property('duration')
 });
 
 App.TezDagVertexState = {

ambari-web/app/routes/main.js

@@ -18,6 +18,7 @@
 
 var App = require('app');
 var stringUtils = require('utils/string_utils');
+var jobsUtils = require('utils/jobs');
 
 module.exports = Em.Route.extend({
   route: '/main',
@@ -111,17 +112,34 @@ module.exports = Em.Route.extend({
     }
   }),
 
-  jobs: Em.Route.extend({
-    route: '/jobs',
-    connectOutlets: function (router) {
-      if (!App.get('isHadoop2Stack')) {
-        Em.run.next(function () {
-          router.transitionTo('main.dashboard');
-        });
-      } else {
-        router.get('mainJobsController').loadJobs();
-        router.get('mainController').connectOutlet('mainJobs');
+  jobs : Em.Route.extend({
+    route : '/jobs',
+    index: Ember.Route.extend({
+      route: '/',
+      connectOutlets : function(router) {
+        if (!App.get('isHadoop2Stack')) {
+          Em.run.next(function() {
+            router.transitionTo('main.dashboard');
+          });
+        } else {
+          router.get('mainJobsController').loadJobs();
+          router.get('mainController').connectOutlet('mainJobs');
+        }
       }
+    }),
+    jobDetails : Em.Route.extend({
+      route : '/:job_id',
+      connectOutlets : function(router, job) {
+        if (job) {
+          jobsUtils.refreshJobDetails(job);
+          if (job.get('jobType') === App.JobType.HIVE) {
+            router.get('mainController').connectOutlet('mainHiveJobDetails', job);
+          }
+        }
+      },
+    }),
+    showJobDetails : function(router, event) {
+      router.transitionTo('jobDetails', event.context);
     }
   }),
 

ambari-web/app/styles/application.less

@@ -5341,3 +5341,32 @@ i.icon-asterisks {
   background-color: #848484;
   color: #FFF;
 }
+
+#hive-job-details {
+  .sections {
+    margin-top: 10px;
+  }
+  #tez-vertices-table-section {
+    display: block;
+  }
+  #tez-vertices-table-container {
+    max-height: 400px;
+    overflow-y: auto;
+  }
+  #tez-vertex-details-section-body {
+    table {
+      margin-bottom: 5px;
+    }
+    td {
+      border-top: none;
+    }
+    tr td:first-child {
+      font-weight: bold;
+      width: 20%
+    }
+    textarea {
+      width: 95%;
+      margin-left: 10px;
+    }
+  }
+}

ambari-web/app/templates/main/jobs.hbs

@@ -53,8 +53,8 @@
         {{else}}
           {{#each job in content}}
             <tr>
-              <td width="40%">
-                {{job.name}}
+              <td>
+                <a title="{{unbound job.name}}" href="#" {{action "showJobDetails" job}}>{{unbound job.name}}</a>
               </td>
               <td>
                 {{job.user}}

ambari-web/app/templates/main/jobs/hive_job_details.hbs

@@ -0,0 +1,107 @@
+{{!
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+}}
+
+<div id="hive-job-details">
+  <!-- Top Bar -->
+  <div class="top-bar">
+    <a {{action "routeToJobs" target="controller"}} href="#">{{t menu.item.jobs}}</a> > {{view.content.name}}
+    <div class="pull-right">Job Type: <span class="label label-info">{{view.content.jobType}}</span></div>
+  </div>
+
+  <!-- Sections -->
+  <div class="row-fluid">
+    <div class="span12 sections">
+      <!-- Section LHS -->
+      <div class="span6 sections-lhs">
+      </div>
+
+      <!-- Section RHS -->
+      <div class="span6 sections-rhs">
+
+        <!-- Section RHS Vertices -->
+        <div id="tez-vertices-table-section">
+          <div id="tez-vertices-table-container" class="section">
+            <table class="table table-hover table-bordered table-striped">
+              <thead>
+                <tr>
+                  <th>{{t common.name}}</th>
+                  <th>{{t common.tasks}}</th>
+                  <th>{{t apps.item.dag.input}}</th>
+                  <th>{{t apps.item.dag.output}}</th>
+                  <th>{{t apps.item.dag.duration}}</th>
+                </tr>
+              </thead>
+              <tbody>
+                {{#each vertex in view.content.tezDag.vertices}}
+                  <tr {{bindAttr class="vertex.isSelected:info"}}>
+                    <td>
+                      <a title="{{vertex.name}}" href="#" {{action "doSelectVertex" vertex target="view"}}>{{vertex.name}}</a>
+                    </td>
+                    <td>{{vertex.tasksCount}}</td>
+                    <td>{{vertex.totalReadBytesDisplay}}</td>
+                    <td>{{vertex.totalWriteBytesDisplay}}</td>
+                    <td>{{vertex.durationDisplay}}</td>
+                  </tr>
+                {{/each}}
+              </tbody>
+            </table>
+          </div>
+        </div>
+
+        <!-- Section RHS Vertex -->
+        {{#if view.selectedVertex}}
+          <div id="section tez-vertex-details-section">
+            <div class="box">
+            <div class="box-header">
+              <h4>{{view.selectedVertex.name}}</h4>
+            </div>
+            <div id="tez-vertex-details-section-body">
+              <table class="table">
+                <tr>
+                  <td>{{t jobs.hive.tez.tasks}}</td>
+                  <td>{{view.selectedVertex.tasksCount}}</td>
+                  <td></td>
+                </tr>
+                <tr>
+                  <td>{{t jobs.hive.tez.hdfs}}</td>
+                  <td>{{view.selectedVertexIODisplay.hdfs.read.ops}} / {{view.selectedVertexIODisplay.hdfs.read.bytes}}</td>
+                  <td>{{view.selectedVertexIODisplay.hdfs.write.ops}} / {{view.selectedVertexIODisplay.hdfs.write.bytes}}</td>
+                </tr>
+                <tr>
+                  <td>{{t jobs.hive.tez.localFiles}}</td>
+                  <td>{{view.selectedVertexIODisplay.file.read.ops}} / {{view.selectedVertexIODisplay.file.read.bytes}}</td>
+                  <td>{{view.selectedVertexIODisplay.file.write.ops}} / {{view.selectedVertexIODisplay.file.write.bytes}}</td>
+                </tr>
+                <tr>
+                  <td>{{t jobs.hive.tez.records}}</td>
+                  <td>{{view.selectedVertexIODisplay.records.read}}</td>
+                  <td>{{view.selectedVertexIODisplay.records.write}}</td>
+                </tr>
+                  <td>{{t jobs.hive.tez.operatorPlan}}</td>
+                  <td></td>
+                  <td></td>
+                </tr>
+              </table>
+              {{view Ember.TextArea valueBinding="view.selectedVertex.operationPlan" rows="15"}}
+            </div>
+          </div>
+        {{/if}}
+      </div>
+    </div>
+  </div>
+</div>

ambari-web/app/utils/ajax.js

@@ -1577,11 +1577,11 @@ var urls = {
     'mock': '/data/jobs/tezDag-name-to-id.json'
   },
   'jobs.tezDag.tezDagId': {
-    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_DAG_ID/{tezDagId}',
+    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_DAG_ID/{tezDagId}?fields=relatedentities',
     'mock': '/data/jobs/tezDag.json'
   },
   'jobs.tezDag.tezDagVertexId': {
-    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_VERTEX_ID/{tezDagVertexId}',
+    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_VERTEX_ID/{tezDagVertexId}?fields=otherinfo',
     'mock': '/data/jobs/tezDagVertex.json'
   }
 };

ambari-web/app/utils/jobs.js

@@ -49,7 +49,7 @@ module.exports = {
     var hiveJobId = hiveJob.get('id');
     // First refresh query
     var hiveQueriesUrl = App.testMode ? "/data/jobs/hive-query-2.json" :
-    App.apiPrefix + "/proxy?url=http://"+historyServerHostName+":8188/ws/v1/apptimeline/HIVE_QUERY_ID/" + hiveJob.get('id');
+    App.apiPrefix + "/proxy?url=http://"+historyServerHostName+":8188/ws/v1/apptimeline/HIVE_QUERY_ID/" + hiveJob.get('id')+"?fields=otherinfo";
     App.HttpClient.get(hiveQueriesUrl, App.hiveJobMapper, {
       complete : function(jqXHR, textStatus) {
         // Now get the Tez DAG ID from the DAG name
@@ -145,6 +145,16 @@ module.exports = {
             vertexRecord.set('tasksCount', data.otherinfo.numTasks);
             vertexRecord.set('state', data.otherinfo.status);
             // TODO Need additional vertex metrics
+            vertexRecord.set('fileReadBytes', 0);
+            vertexRecord.set('fileReadOps', 0);
+            vertexRecord.set('fileWriteOps', 0);
+            vertexRecord.set('fileWriteBytes', 0);
+            vertexRecord.set('hdfsReadOps', 0);
+            vertexRecord.set('hdfsReadBytes', 0);
+            vertexRecord.set('hdfsWriteOps', 0);
+            vertexRecord.set('hdfsWriteBytes', 0);
+            vertexRecord.set('recordReadCount', 0);
+            vertexRecord.set('recordWriteCount', 0);
           }
         }
       },

ambari-web/app/views.js

@@ -212,6 +212,7 @@ require('views/main/charts/heatmap/heatmap_host');
 require('views/main/charts/heatmap/heatmap_host_detail');
 require('views/main/apps_view');
 require('views/main/jobs_view');
+require('views/main/jobs/hive_job_details_view');
 require('views/main/apps/item_view');
 require('views/main/apps/item/bar_view');
 require('views/main/apps/item/dag_view');

ambari-web/app/views/main/jobs/hive_job_details_view.js

@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+var date = require('utils/date');
+var numberUtils = require('utils/number_utils');
+
+App.MainHiveJobDetailsView = Em.View.extend({
+  templateName : require('templates/main/jobs/hive_job_details'),
+
+  selectedVertex : null,
+
+  content : function() {
+    return this.get('controller.content');
+  }.property('controller.content'),
+
+  jobObserver : function() {
+    var content = this.get('content');
+    var selectedVertex = this.get('selectedVertex');
+    if (selectedVertex == null && content != null) {
+      var vertices = content.get('tezDag.vertices');
+      if (vertices) {
+        vertices.setEach('isSelected', false);
+        this.doSelectVertex({context:vertices.objectAt(0)});
+      }
+    }
+  }.observes('selectedVertex', 'content.tezDag.vertices.@each.id'),
+
+  doSelectVertex : function(event) {
+    var newVertex = event.context;
+    var currentVertex = this.get('selectedVertex');
+    if (currentVertex != null) {
+      currentVertex.set('isSelected', false);
+    }
+    newVertex.set('isSelected', true);
+    this.set('selectedVertex', newVertex);
+  },
+
+  /**
+   * Provides display information for vertex I/O.
+   * 
+   * {
+   *  'file': {
+   *    'read': {
+   *      'ops': '100 reads',
+   *      'bytes': '10 MB'
+   *    }
+   *    'write: {
+   *      'ops': '200 writes',
+   *      'bytes': '20 MB'
+   *    }
+   *  },
+   *  'hdfs': {
+   *    'read': {
+   *      'ops': '100 reads',
+   *      'bytes': '10 MB'
+   *    }
+   *    'write: {
+   *      'ops': '200 writes',
+   *      'bytes': '20 MB'
+   *    }
+   *  },
+   *  'records': {
+   *    'read': '100 records',
+   *    'write': '123 records'
+   *  }
+   * }
+   */
+  selectedVertexIODisplay : function() {
+    var v = this.get('selectedVertex');
+    return {
+      file : {
+        read : {
+          ops : Em.I18n.t('jobs.hive.tez.reads').format(v.get('fileReadOps')),
+          bytes : numberUtils.bytesToSize(v.get('fileReadBytes'))
+        },
+        write : {
+          ops : Em.I18n.t('jobs.hive.tez.writes').format(v.get('fileWriteOps')),
+          bytes : numberUtils.bytesToSize(v.get('fileWriteBytes'))
+        }
+      },
+      hdfs : {
+        read : {
+          ops : Em.I18n.t('jobs.hive.tez.reads').format(v.get('hdfsReadOps')),
+          bytes : numberUtils.bytesToSize(v.get('hdfsReadBytes'))
+        },
+        write : {
+          ops : Em.I18n.t('jobs.hive.tez.writes').format(v.get('hdfsWriteOps')),
+          bytes : numberUtils.bytesToSize(v.get('hdfsWriteBytes'))
+        }
+      },
+      records : {
+        read : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordReadCount')),
+        write : Em.I18n.t('jobs.hive.tez.records.count').format(v.get('recordWriteCount')),
+      }
+    };
+  }.property('selectedVertex.fileReadOps', 'selectedVertex.fileWriteOps', 'selectedVertex.hdfsReadOps', 'selectedVertex.hdfdWriteOps',
+      'selectedVertex.fileReadBytes', 'selectedVertex.fileWriteBytes', 'selectedVertex.hdfsReadBytes', 'selectedVertex.hdfdWriteBytes',
+      'selectedVertex.recordReadCount', 'selectedVertes.recordWriteCount')
+});