AMBARI-5080. Job Tez DAG should handle a union query. (srimanth)

ambari-web/app/mappers/jobs/hive_job_mapper.js

@@ -95,13 +95,15 @@ App.hiveJobMapper = App.QuickDataMapper.create({
               return ops;
             }
             if (vertex["Map Operator Tree:"] != null) {
-              vertexObj.is_map = true;
+              vertexObj.type = App.TezDagVertexType.MAP;
               vertexObj.operations = operatorExtractor(vertex["Map Operator Tree:"]);
               vertexObj.operation_plan = JSON.stringify(vertex["Map Operator Tree:"], undefined, "  ");
             } else if (vertex["Reduce Operator Tree:"] != null) {
-              vertexObj.is_map = false;
+              vertexObj.type = App.TezDagVertexType.REDUCE;
               vertexObj.operations = operatorExtractor(vertex["Reduce Operator Tree:"]);
               vertexObj.operation_plan = JSON.stringify(vertex["Reduce Operator Tree:"], undefined, "  ");
+            } else if (vertex["Vertex:"] != null && vertexName==vertex['Vertex:']) {
+              vertexObj.type = App.TezDagVertexType.UNION;
             }
             vertexIdMap[vertexObj.id] = vertexObj;
             vertices.push(vertexObj);
@@ -117,6 +119,15 @@ App.hiveJobMapper = App.QuickDataMapper.create({
             }
             childVertices.forEach(function(e) {
               var parentVertex = e.parent;
+              if (e.type == 'CONTAINS') {
+                var parentVertexNode = vertexIdMap[dagName + "/" + parentVertex];
+                if (parentVertexNode != null && parentVertexNode.type == App.TezDagVertexType.UNION) {
+                  // We flip the edges for Union vertices
+                  var tmp = childVertex;
+                  childVertex = parentVertex;
+                  parentVertex = tmp;
+                }
+              }
               var edgeObj = {
                 id : dagName + "/" + parentVertex + "-" + childVertex,
                 from_vertex_id : dagName + "/" + parentVertex,
@@ -127,10 +138,13 @@ App.hiveJobMapper = App.QuickDataMapper.create({
               edgeIds.push(edgeObj.id);
               switch (e.type) {
               case "BROADCAST_EDGE":
-                edgeObj.edge_type = App.TezDagVertexType.BROADCAST;
+                edgeObj.edge_type = App.TezDagEdgeType.BROADCAST;
                 break;
               case "SIMPLE_EDGE":
-                edgeObj.edge_type = App.TezDagVertexType.SCATTER_GATHER;
+                edgeObj.edge_type = App.TezDagEdgeType.SCATTER_GATHER;
+                break;
+              case "CONTAINS":
+                edgeObj.edge_type = App.TezDagEdgeType.CONTAINS;
                 break;
               default:
                 break;

ambari-web/app/messages.js

@@ -1970,6 +1970,8 @@ Em.I18n.translations = {
   'jobs.hive.tez.metric.recordsRead':'Records Read',
   'jobs.hive.tez.metric.recordsWrite':'Records Written',
   'jobs.hive.tez.metric.tezTasks':'Tez Tasks',
+  'jobs.hive.tez.edge.':'Unknown',
+  'jobs.hive.tez.edge.contains':'Contains',
   'jobs.hive.tez.edge.broadcast':'Broadcast',
   'jobs.hive.tez.edge.scatter_gather':'Shuffle',
 

ambari-web/app/models/jobs/tez_dag.js

@@ -38,7 +38,7 @@ App.TezDagEdge = DS.Model.extend({
   toVertex : DS.belongsTo('App.TezDagVertex'),
   /**
    * Type of this edge connecting vertices. Should be one of constants defined
-   * in 'App.TezDagVertexType'.
+   * in 'App.TezDagEdgeType'.
    */
   edgeType : DS.attr('string')
 });
@@ -58,9 +58,10 @@ App.TezDagVertex = DS.Model.extend({
   state : DS.attr('string'),
 
   /**
-   * @return {Boolean} Whether this vertex is a Map or Reduce operation.
+   * Vertex type has to be one of the types defined in 'App.TezDagVertexType'
+   * @return {string}
    */
-  isMap : DS.attr('boolean'),
+  type : DS.attr('string'),
 
   /**
    * A vertex can have multiple incoming edges.
@@ -158,8 +159,15 @@ App.TezDagVertexState = {
 };
 
 App.TezDagVertexType = {
+  MAP: 'MAP',
+  REDUCE: 'REDUCE',
+  UNION: 'UNION'
+};
+
+App.TezDagEdgeType = {
   SCATTER_GATHER : "SCATTER_GATHER",
-  BROADCAST : "BROADCAST"
+  BROADCAST : "BROADCAST",
+  CONTAINS: "CONTAINS"
 };
 
 App.TezDag.FIXTURES = [];

ambari-web/app/styles/application.less

@@ -5796,6 +5796,14 @@ i.icon-asterisks {
         fill: #DBD4EB;
         stroke: #bbbbbb;
       }
+      .background.union {
+        fill: #BCE6DD;
+        stroke: #bbbbbb;
+      }
+      .background.unknown-vertex-type {
+        fill: #EBE8CC;
+        stroke: #bbbbbb;
+      }
       .background.selected {
         stroke: #3a87ad;
       }
@@ -5860,9 +5868,16 @@ i.icon-asterisks {
     path.link.type-broadcast {
       stroke: teal;
     }
-    path.link.type-scatter-gather {
+    path.link.type-scatter_gather {
       stroke: orange;
       stroke-dasharray: 10,3;
     }
+    path.link.type-contains {
+      stroke: green;
+      stroke-dasharray: 10,2,5,2;
+    }
+    path.link.type-unknown {
+      stroke: gray;
+    }
   }
 }

ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js

@@ -44,7 +44,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
    *     {
    *       "id": "Map2",
    *       "name": "Map 2",
-   *       "isMap": true,
+   *       "type": App.TezDagVertexType.MAP,
    *       "operations": [
    *         "TableScan",
    *         "File Output"
@@ -304,7 +304,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
           id : vertex.get('id'),
           name : vertex.get('name'),
           state : vertex.get('state'),
-          isMap : vertex.get('isMap'),
+          type : vertex.get('type'),
           operations : vertex.get('operations'),
           depth : edgeObj.depth,
           parents : [],
@@ -558,15 +558,10 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
     var link = svgLayer.selectAll(".link-g").data(dagVisualModel.links).enter().append("g").attr("class", "link-g").attr("marker-end", "url(#arrow)");
     link.append("path").attr("class", function(l) {
       var classes = "link svg-tooltip ";
-      switch (l.edgeType) {
-      case App.TezDagVertexType.BROADCAST:
-        classes += "type-broadcast ";
-        break;
-      case App.TezDagVertexType.SCATTER_GATHER:
-        classes += "type-scatter-gather ";
-        break;
-      default:
-        break;
+      if (l.edgeType) {
+        classes += ("type-" + l.edgeType.toLowerCase() + " ");
+      } else {
+        classes += "type-unknown ";
       }
       return classes;
     }).attr("d", diagonal).attr("title", function(l) {
@@ -590,79 +585,91 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
     node.each(function(n, nodeIndex) {
       var ops = n.operations;
       var opCount = {};
-      var opGroups = d3.select(this).selectAll(".operation").data(ops).enter().append("g").attr("class", "operation").attr("transform", function(op, opIndex) {
-        var row = Math.floor(opIndex / 3);
-        var column = opIndex % 3;
-        return "translate(" + (10 + column * 55) + "," + (37 + row * 20) + ")";
-      }).attr("clip-path", "url(#operatorClipPath)").attr("opIndex", function(op){
-        if(!opCount[op]) {
-          opCount[op] = 1;
-        } else {
-          opCount[op] = opCount[op]+1;
-        }
-        return opCount[op];
-      }).on('mousedown', function(op) {
-        var opIndex = this.getAttribute ? this.getAttribute("opIndex") : null;
-        if (numberUtils.validateInteger(opIndex) == null) {
-          console.log("Clicked on operator: ", op, " [", opIndex, "]");
-          var textArea = document.getElementById('tez-vertex-operator-plan-textarea');
-          if (textArea && textArea.value) {
-            var text = textArea.value;
-            var opText = "\"" + op + "\"";
-            var count = 1;
-            var index = text.indexOf(opText);
-            while (index > -1 && count < opIndex) {
-              index = text.indexOf(opText, index + 1);
-              count++;
-            }
-            if (index > -1) {
-              var start = index;
-              var end = index;
-              var matchCount = 0;
-              var splits = text.substring(start).split(/({|})/);
-              splits.every(function(s) {
-                if (s == '{') {
-                  matchCount++;
-                } else if (s == '}') {
-                  matchCount--;
-                  if (matchCount == 0) {
-                    end += s.length;
-                    return false;
+      if (ops != null && ops.length > 0) {
+        var opGroups = d3.select(this).selectAll(".operation").data(ops).enter().append("g").attr("class", "operation").attr("transform", function(op, opIndex) {
+          var row = Math.floor(opIndex / 3);
+          var column = opIndex % 3;
+          return "translate(" + (10 + column * 55) + "," + (37 + row * 20) + ")";
+        }).attr("clip-path", "url(#operatorClipPath)").attr("opIndex", function(op){
+          if(!opCount[op]) {
+            opCount[op] = 1;
+          } else {
+            opCount[op] = opCount[op]+1;
+          }
+          return opCount[op];
+        }).on('mousedown', function(op) {
+          var opIndex = this.getAttribute ? this.getAttribute("opIndex") : null;
+          if (numberUtils.validateInteger(opIndex) == null) {
+            console.log("Clicked on operator: ", op, " [", opIndex, "]");
+            var textArea = document.getElementById('tez-vertex-operator-plan-textarea');
+            if (textArea && textArea.value) {
+              var text = textArea.value;
+              var opText = "\"" + op + "\"";
+              var count = 1;
+              var index = text.indexOf(opText);
+              while (index > -1 && count < opIndex) {
+                index = text.indexOf(opText, index + 1);
+                count++;
+              }
+              if (index > -1) {
+                var start = index;
+                var end = index;
+                var matchCount = 0;
+                var splits = text.substring(start).split(/({|})/);
+                splits.every(function(s) {
+                  if (s == '{') {
+                    matchCount++;
+                  } else if (s == '}') {
+                    matchCount--;
+                    if (matchCount == 0) {
+                      end += s.length;
+                      return false;
+                    }
                   }
+                  end += s.length;
+                  return true;
+                });
+                textArea.setSelectionRange(start, end);
+                // Now scroll to the selection
+                var lines = 0;
+                var totalLines = 0;
+                var index = text.indexOf("\n");
+                while (index > 0) {
+                  index = text.indexOf("\n", index + 1);
+                  if (index < start) {
+                    lines++;
+                  }
+                  totalLines++;
                 }
-                end += s.length;
-                return true;
-              });
-              textArea.setSelectionRange(start, end);
-              // Now scroll to the selection
-              var lines = 0;
-              var totalLines = 0;
-              var index = text.indexOf("\n");
-              while (index > 0) {
-                index = text.indexOf("\n", index + 1);
-                if (index < start) {
-                  lines++;
-                }
-                totalLines++;
+                console.log("Selection is from row ", lines, " out of ", totalLines);
+                lines -= 5;
+                var lineHeight = Math.floor(textArea.scrollHeight / totalLines);
+                var scrollHeight = Math.round(lines * lineHeight);
+                textArea.scrollTop = scrollHeight;
               }
-              console.log("Selection is from row ", lines, " out of ", totalLines);
-              lines -= 5;
-              var lineHeight = Math.floor(textArea.scrollHeight / totalLines);
-              var scrollHeight = Math.round(lines * lineHeight);
-              textArea.scrollTop = scrollHeight;
             }
           }
-        }
-      });
-      opGroups.append("rect").attr("class", "operation svg-tooltip ").attr("width", "50").attr("height", "16").attr("title", function(op) {
-        return op;
-      });
-      opGroups.append("text").attr("x", "2").attr("dy", "1em").text(function(op) {
-        return op != null ? op.split(' ')[0] : '';
-      });
+        });
+        opGroups.append("rect").attr("class", "operation svg-tooltip ").attr("width", "50").attr("height", "16").attr("title", function(op) {
+          return op;
+        });
+        opGroups.append("text").attr("x", "2").attr("dy", "1em").text(function(op) {
+          return op != null ? op.split(' ')[0] : '';
+        });
+      }
     });
     var metricNodes = node.append("g").attr("class", "metric").attr("transform", "translate(112,7)");
-    metricNodes.append("rect").attr("width", 60).attr("height", 18).attr("rx", "3").attr("class", "metric-title svg-tooltip");
+    metricNodes.append("rect").attr("width", function(n) {
+      if (n.type == App.TezDagVertexType.UNION) {
+        return 0;
+      }
+      return 60;
+    }).attr("height", function(n) {
+      if (n.type == App.TezDagVertexType.UNION) {
+        return 0;
+      }
+      return 18;
+    }).attr("rx", "3").attr("class", "metric-title svg-tooltip");
     metricNodes.append("text").attr("class", "metric-text").attr("x", "2").attr("dy", "1em");
     node.append("text").attr("x", "1.9em").attr("dy", "1.5em").text(function(d) {
       return d.name;
@@ -757,6 +764,9 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
         return classes;
       });
       metricNodeTexts.text(function(node){
+        if (node.type == App.TezDagVertexType.UNION) {
+          return '';
+        }
         return node.metricDisplay;
       });
       metricNodeTitles.attr("title", function(node){
@@ -766,10 +776,10 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       });
       nodeBackgrounds.attr("class", function(n) {
         var classes = "background ";
-        if (n.isMap) {
-          classes += "map ";
+        if (n.type) {
+          classes += (n.type.toLowerCase() + " ");
         } else {
-          classes += "reduce ";
+          classes += "unknown-vertex-type ";
         }
         if (n.selected) {
           classes += "selected ";
@@ -829,7 +839,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       width : 180,
       height : 40
     }
-    if (node.operations.length > 0) {
+    if (node.operations && node.operations.length > 0) {
       var opsHeight = Math.ceil(node.operations.length / 3);
       size.height += (opsHeight * 20);
     }

ambari-web/app/views/main/jobs/hive_job_details_view.js

@@ -166,24 +166,41 @@ App.MainHiveJobDetailsView = Em.View.extend({
     if (status) {
       status = stringUtils.getCamelCase(status);
     }
+    var fileReadOps = v.get('fileReadOps');
+    var fileWriteOps = v.get('fileWriteOps');
+    var hdfsReadOps = v.get('hdfsReadOps');
+    var hdfsWriteOps = v.get('hdfsWriteOps');
+    var naString = Em.I18n.t('common.na');
+    if (fileReadOps === null) {
+      fileReadOps = naString;
+    }
+    if (fileWriteOps === null) {
+      fileWriteOps = naString;
+    }
+    if (hdfsReadOps === null) {
+      hdfsReadOps = naString;
+    }
+    if (hdfsWriteOps === null) {
+      hdfsWriteOps = naString;
+    }
     return {
       file : {
         read : {
-          ops : Em.I18n.t('jobs.hive.tez.reads').format(v.get('fileReadOps')),
+          ops : Em.I18n.t('jobs.hive.tez.reads').format(fileReadOps),
           bytes : numberUtils.bytesToSize(v.get('fileReadBytes'))
         },
         write : {
-          ops : Em.I18n.t('jobs.hive.tez.writes').format(v.get('fileWriteOps')),
+          ops : Em.I18n.t('jobs.hive.tez.writes').format(fileWriteOps),
           bytes : numberUtils.bytesToSize(v.get('fileWriteBytes'))
         }
       },
       hdfs : {
         read : {
-          ops : Em.I18n.t('jobs.hive.tez.reads').format(v.get('hdfsReadOps')),
+          ops : Em.I18n.t('jobs.hive.tez.reads').format(hdfsReadOps),
           bytes : numberUtils.bytesToSize(v.get('hdfsReadBytes'))
         },
         write : {
-          ops : Em.I18n.t('jobs.hive.tez.writes').format(v.get('hdfsWriteOps')),
+          ops : Em.I18n.t('jobs.hive.tez.writes').format(hdfsWriteOps),
           bytes : numberUtils.bytesToSize(v.get('hdfsWriteBytes'))
         }
       },

ambari-web/test/views/main/jobs/hive_job_details_tez_test.js

@@ -17,13 +17,13 @@
 
 var App = require('app');
 module.exports = {
-  _createVertex : function(row, col, state, isMap, numOps, inEdges, outEdges, vertexJsons) {
+  _createVertex : function(row, col, state, type, numOps, inEdges, outEdges, vertexJsons) {
     var v = {
       id : 'v_' + row + '_' + col,
       instance_id : 'vi_' + row + '_' + col,
       name : 'Vertex ' + row + ', ' + col,
       state : state,
-      is_map : isMap,
+      type : type,
       operations : [],
       outgoing_edges : outEdges,
       incoming_edges : inEdges
@@ -70,20 +70,20 @@ module.exports = {
     var vertexJsons = [];
     var edgeJsons = [];
     // Row 1
-    var v1 = this._createVertex(1, 1, "FAILED", true, 30, [], [ 'e1' ], vertexJsons);
-    var v2 = this._createVertex(1, 2, "RUNNING", true, 2, [], [ 'e2' ], vertexJsons);
-    var v3 = this._createVertex(1, 3, "FAILED", true, 5, [], [ 'e3' ], vertexJsons);
-    var v4 = this._createVertex(1, 4, "FAILED", true, 10, [], [ 'e4' ], vertexJsons);
-    var v5 = this._createVertex(1, 5, "FAILED", true, 15, [], [ 'e5' ], vertexJsons);
-    var v6 = this._createVertex(1, 6, "FAILED", true, 20, [], [ 'e6' ], vertexJsons);
+    var v1 = this._createVertex(1, 1, "FAILED", App.TezDagVertexType.MAP, 30, [], [ 'e1' ], vertexJsons);
+    var v2 = this._createVertex(1, 2, "RUNNING", App.TezDagVertexType.REDUCE, 2, [], [ 'e2' ], vertexJsons);
+    var v3 = this._createVertex(1, 3, "FAILED", App.TezDagVertexType.MAP, 5, [], [ 'e3' ], vertexJsons);
+    var v4 = this._createVertex(1, 4, "FAILED", App.TezDagVertexType.REDUCE, 10, [], [ 'e4' ], vertexJsons);
+    var v5 = this._createVertex(1, 5, "FAILED", App.TezDagVertexType.MAP, 15, [], [ 'e5' ], vertexJsons);
+    var v6 = this._createVertex(1, 6, "FAILED", App.TezDagVertexType.REDUCE, 20, [], [ 'e6' ], vertexJsons);
     // Row 2
-    var v7 = this._createVertex(2, 1, "SUCCEEDED", false, 30, [ 'e1', 'e2', 'e3', 'e4', 'e5', 'e6' ], [ 'e7', 'e8', 'e9', 'e10', 'e11' ], vertexJsons);
+    var v7 = this._createVertex(2, 1, "SUCCEEDED", App.TezDagVertexType.UNION, 30, [ 'e1', 'e2', 'e3', 'e4', 'e5', 'e6' ], [ 'e7', 'e8', 'e9', 'e10', 'e11' ], vertexJsons);
     // Row 3
-    var v8 = this._createVertex(3, 1, "FAILED", false, 30, [ 'e7' ], [], vertexJsons);
-    var v9 = this._createVertex(3, 2, "RUNNING", false, 2, [ 'e8' ], [], vertexJsons);
-    var v10 = this._createVertex(3, 3, "FAILED", false, 5, [ 'e9' ], [], vertexJsons);
-    var v11 = this._createVertex(3, 4, "FAILED", true, 10, [ 'e10' ], [], vertexJsons);
-    var v12 = this._createVertex(3, 5, "FAILED", true, 15, [ 'e11' ], [], vertexJsons);
+    var v8 = this._createVertex(3, 1, "FAILED", App.TezDagVertexType.REDUCE, 30, [ 'e7' ], [], vertexJsons);
+    var v9 = this._createVertex(3, 2, "RUNNING", App.TezDagVertexType.MAP, 2, [ 'e8' ], [], vertexJsons);
+    var v10 = this._createVertex(3, 3, "FAILED", App.TezDagVertexType.REDUCE, 5, [ 'e9' ], [], vertexJsons);
+    var v11 = this._createVertex(3, 4, "FAILED", App.TezDagVertexType.MAP, 10, [ 'e10' ], [], vertexJsons);
+    var v12 = this._createVertex(3, 5, "FAILED", App.TezDagVertexType.REDUCE, 15, [ 'e11' ], [], vertexJsons);
     // Edges 1-2
     this._createEdge('e1', 'BROADCAST', v1, v7, edgeJsons);
     this._createEdge('e2', 'BROADCAST', v2, v7, edgeJsons);
@@ -132,18 +132,18 @@ module.exports = {
     var vertexJsons = [];
     var edgeJsons = [];
     // Row 1
-    var v1 = this._createVertex(1, 1, "FAILED", true, 30, [], [ 'e1' ], vertexJsons);
-    var v4 = this._createVertex(1, 4, "FAILED", true, 10, [], [ 'e4' ], vertexJsons);
-    var v6 = this._createVertex(1, 6, "FAILED", true, 20, [], [ 'e6' ], vertexJsons);
-    var v2 = this._createVertex(1, 2, "RUNNING", true, 2, [], [ 'e2' ], vertexJsons);
-    var v3 = this._createVertex(1, 3, "FAILED", true, 5, [], [ 'e3' ], vertexJsons);
-    var v5 = this._createVertex(1, 5, "FAILED", true, 15, [], [ 'e5' ], vertexJsons);
-    var v7 = this._createVertex(1, 7, "FAILED", true, 4, [], [ 'e7' ], vertexJsons);
+    var v1 = this._createVertex(1, 1, "FAILED", App.TezDagVertexType.REDUCE, 30, [], [ 'e1' ], vertexJsons);
+    var v4 = this._createVertex(1, 4, "FAILED", App.TezDagVertexType.MAP, 10, [], [ 'e4' ], vertexJsons);
+    var v6 = this._createVertex(1, 6, "FAILED", App.TezDagVertexType.REDUCE, 20, [], [ 'e6' ], vertexJsons);
+    var v2 = this._createVertex(1, 2, "RUNNING", App.TezDagVertexType.MAP, 2, [], [ 'e2' ], vertexJsons);
+    var v3 = this._createVertex(1, 3, "FAILED", App.TezDagVertexType.REDUCE, 5, [], [ 'e3' ], vertexJsons);
+    var v5 = this._createVertex(1, 5, "FAILED", App.TezDagVertexType.MAP, 15, [], [ 'e5' ], vertexJsons);
+    var v7 = this._createVertex(1, 7, "FAILED", App.TezDagVertexType.REDUCE, 4, [], [ 'e7' ], vertexJsons);
     // Row 2
-    var v8 = this._createVertex(2, 1, "SUCCEEDED", false, 30, [ 'e1', 'e2', 'e3', 'e4' ], [ 'e8' ], vertexJsons);
-    var v9 = this._createVertex(2, 2, "FAILED", false, 30, [ 'e5', 'e6', 'e7' ], ['e9'], vertexJsons);
+    var v8 = this._createVertex(2, 1, "SUCCEEDED", App.TezDagVertexType.MAP, 30, [ 'e1', 'e2', 'e3', 'e4' ], [ 'e8' ], vertexJsons);
+    var v9 = this._createVertex(2, 2, "FAILED", App.TezDagVertexType.REDUCE, 30, [ 'e5', 'e6', 'e7' ], ['e9'], vertexJsons);
     // Row 3
-    var v10 = this._createVertex(3, 1, "RUNNING", false, 2, [ 'e8', 'e9' ], [], vertexJsons);
+    var v10 = this._createVertex(3, 1, "RUNNING", App.TezDagVertexType.UNION, 2, [ 'e8', 'e9' ], [], vertexJsons);
     // Edges 1-2
     this._createEdge('e1', 'BROADCAST', v1, v8, edgeJsons);
     this._createEdge('e2', 'BROADCAST', v2, v8, edgeJsons);