AMBARI-5037. Displaced vertex in the hive query layout. (srimanth)

ambari-web/app/assets/data/jobs/tezDag.json

@@ -1,15 +1,18 @@
 {
-  "starttime": 1393031617162,
-  "entity": "dag_1393012407618_0059_3",
+  "starttime": 1394569228227,
+  "entity": "dag_1394502141829_0425_1",
   "entitytype": "TEZ_DAG_ID",
   "relatedentities": {
     "TEZ_VERTEX_ID": [
-      "vertex_1393012407618_0059_3_01",
-      "vertex_1393012407618_0059_3_00",
-      "vertex_1393012407618_0059_3_03",
-      "vertex_1393012407618_0059_3_02",
-      "vertex_1393012407618_0059_3_05",
-      "vertex_1393012407618_0059_3_04"
+      "vertex_1394502141829_0425_1_00",
+      "vertex_1394502141829_0425_1_01",
+      "vertex_1394502141829_0425_1_02",
+      "vertex_1394502141829_0425_1_03",
+      "vertex_1394502141829_0425_1_04",
+      "vertex_1394502141829_0425_1_05",
+      "vertex_1394502141829_0425_1_06",
+      "vertex_1394502141829_0425_1_07",
+      "vertex_1394502141829_0425_1_08"
     ]
   }
 }

ambari-web/app/assets/data/jobs/tezDagVertex.json

@@ -1,15 +1,15 @@
 {
-  "starttime": 1393031617901,
+  "starttime": 1394569231815,
   "otherinfo": {
-    "startTime": 1393031617902,
-    "status": "SUCCEEDED",
-    "numTasks": 170,
-    "initTime": 1393031617901,
+    "startTime": 1394569231819,
+    "status": "KILLED",
+    "numTasks": 6,
+    "initTime": 1394569231815,
     "vertexName": "Map 2",
-    "timeTaken": 6352,
+    "timeTaken": 71735,
     "processorClassName": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
-    "endTime": 1393031624254,
-    "initRequestedTime": 1393031617436,
+    "endTime": 1394569303554,
+    "initRequestedTime": 1394569230142,
     "counters": {
       "counterGroups": [
         {
@@ -19,12 +19,7 @@
             {
               "counterName": "DATA_LOCAL_TASKS",
               "counterDisplayName": "DATA_LOCAL_TASKS",
-              "counterValue": 155
-            },
-            {
-              "counterName": "RACK_LOCAL_TASKS",
-              "counterDisplayName": "RACK_LOCAL_TASKS",
-              "counterValue": 15
+              "counterValue": 4
             }
           ]
         },
@@ -40,7 +35,7 @@
             {
               "counterName": "FILE_BYTES_WRITTEN",
               "counterDisplayName": "FILE: BYTES_WRITTEN",
-              "counterValue": 6141674
+              "counterValue": 337559
             },
             {
               "counterName": "FILE_READ_OPS",
@@ -60,7 +55,7 @@
             {
               "counterName": "HDFS_BYTES_READ",
               "counterDisplayName": "HDFS: BYTES_READ",
-              "counterValue": 1291299199
+              "counterValue": 12854749
             },
             {
               "counterName": "HDFS_BYTES_WRITTEN",
@@ -70,7 +65,7 @@
             {
               "counterName": "HDFS_READ_OPS",
               "counterDisplayName": "HDFS: READ_OPS",
-              "counterValue": 14094
+              "counterValue": 12
             },
             {
               "counterName": "HDFS_LARGE_READ_OPS",
@@ -89,83 +84,138 @@
           "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
           "counters": [
             {
-              "counterName": "MAP_INPUT_RECORDS",
-              "counterDisplayName": "MAP_INPUT_RECORDS",
-              "counterValue": 108378
-            },
-            {
-              "counterName": "MAP_OUTPUT_RECORDS",
-              "counterDisplayName": "MAP_OUTPUT_RECORDS",
-              "counterValue": 394097
-            },
-            {
-              "counterName": "MAP_OUTPUT_BYTES",
-              "counterDisplayName": "MAP_OUTPUT_BYTES",
-              "counterValue": 34555222
+              "counterName": "SPILLED_RECORDS",
+              "counterDisplayName": "SPILLED_RECORDS",
+              "counterValue": 7194
             },
             {
-              "counterName": "MAP_OUTPUT_MATERIALIZED_BYTES",
-              "counterDisplayName": "MAP_OUTPUT_MATERIALIZED_BYTES",
-              "counterValue": 6136234
+              "counterName": "NUM_SHUFFLED_INPUTS",
+              "counterDisplayName": "NUM_SHUFFLED_INPUTS",
+              "counterValue": 16
             },
             {
-              "counterName": "SPILLED_RECORDS",
-              "counterDisplayName": "SPILLED_RECORDS",
-              "counterValue": 394097
+              "counterName": "NUM_FAILED_SHUFFLE_INPUTS",
+              "counterDisplayName": "NUM_FAILED_SHUFFLE_INPUTS",
+              "counterValue": 0
             },
             {
               "counterName": "GC_TIME_MILLIS",
               "counterDisplayName": "GC_TIME_MILLIS",
-              "counterValue": 9246
+              "counterValue": 2047
             },
             {
               "counterName": "CPU_MILLISECONDS",
               "counterDisplayName": "CPU_MILLISECONDS",
-              "counterValue": -2742060
+              "counterValue": -11970
             },
             {
               "counterName": "PHYSICAL_MEMORY_BYTES",
               "counterDisplayName": "PHYSICAL_MEMORY_BYTES",
-              "counterValue": 232119492608
+              "counterValue": 1791279104
             },
             {
               "counterName": "VIRTUAL_MEMORY_BYTES",
               "counterDisplayName": "VIRTUAL_MEMORY_BYTES",
-              "counterValue": 793722957824
+              "counterValue": 7152783360
             },
             {
               "counterName": "COMMITTED_HEAP_BYTES",
               "counterDisplayName": "COMMITTED_HEAP_BYTES",
-              "counterValue": 262738477056
+              "counterValue": 1588592640
+            },
+            {
+              "counterName": "INPUT_RECORDS_PROCESSED",
+              "counterDisplayName": "INPUT_RECORDS_PROCESSED",
+              "counterValue": 1990495
+            },
+            {
+              "counterName": "OUTPUT_RECORDS",
+              "counterDisplayName": "OUTPUT_RECORDS",
+              "counterValue": 7194
+            },
+            {
+              "counterName": "OUTPUT_BYTES",
+              "counterDisplayName": "OUTPUT_BYTES",
+              "counterValue": 323019
+            },
+            {
+              "counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
+              "counterDisplayName": "OUTPUT_BYTES_WITH_OVERHEAD",
+              "counterValue": 337415
+            },
+            {
+              "counterName": "OUTPUT_BYTES_PHYSICAL",
+              "counterDisplayName": "OUTPUT_BYTES_PHYSICAL",
+              "counterValue": 337431
+            },
+            {
+              "counterName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+              "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+              "counterValue": 0
+            },
+            {
+              "counterName": "ADDITIONAL_SPILLS_BYTES_READ",
+              "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_READ",
+              "counterValue": 0
+            },
+            {
+              "counterName": "ADDITIONAL_SPILL_COUNT",
+              "counterDisplayName": "ADDITIONAL_SPILL_COUNT",
+              "counterValue": 0
+            },
+            {
+              "counterName": "SHUFFLE_BYTES",
+              "counterDisplayName": "SHUFFLE_BYTES",
+              "counterValue": 6124044
+            },
+            {
+              "counterName": "SHUFFLE_BYTES_DECOMPRESSED",
+              "counterDisplayName": "SHUFFLE_BYTES_DECOMPRESSED",
+              "counterValue": 6123980
+            },
+            {
+              "counterName": "SHUFFLE_BYTES_TO_MEM",
+              "counterDisplayName": "SHUFFLE_BYTES_TO_MEM",
+              "counterValue": 6124044
+            },
+            {
+              "counterName": "SHUFFLE_BYTES_TO_DISK",
+              "counterDisplayName": "SHUFFLE_BYTES_TO_DISK",
+              "counterValue": 0
             }
           ]
         },
         {
-          "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
-          "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counterGroupName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
+          "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
           "counters": [
             {
-              "counterName": "DESERIALIZE_ERRORS",
-              "counterDisplayName": "DESERIALIZE_ERRORS",
+              "counterName": "FILTERED",
+              "counterDisplayName": "FILTERED",
               "counterValue": 0
+            },
+            {
+              "counterName": "PASSED",
+              "counterDisplayName": "PASSED",
+              "counterValue": 95911
             }
           ]
         },
         {
-          "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
-          "counterGroupDisplayName": "File Input Format Counters ",
+          "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
           "counters": [
             {
-              "counterName": "BYTES_READ",
-              "counterDisplayName": "Bytes Read",
+              "counterName": "DESERIALIZE_ERRORS",
+              "counterDisplayName": "DESERIALIZE_ERRORS",
               "counterValue": 0
             }
           ]
         }
       ]
     },
-    "startRequestedTime": 1393031617487
+    "startRequestedTime": 1394569231792
   },
-  "entity": "vertex_1393012407618_0059_3_01",
+  "entity": "vertex_1394502141829_0425_1_04",
   "entitytype": "TEZ_VERTEX_ID"
 }

ambari-web/app/views/main/jobs/hive_job_details_tez_dag_view.js

@@ -290,6 +290,10 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       var pName = edgeObj.parent ? edgeObj.parent.name : null;
       var cName = edgeObj.toVertex ? edgeObj.toVertex.get('name') : null;
       console.debug("Processing vertex ", edgeObj, " (",pName, " > ", cName,")");
+      if(edgeObj.parent && edgeObj.depth < edgeObj.parent.depth + 1){
+        console.debug("Updating child edge to " + (edgeObj.parent.depth + 1));
+        edgeObj.depth = edgeObj.parent.depth + 1;
+      }
       var node = vertexIdToNode[vertex.get('id')];
       for ( var k = depthToNodes.length; k <= edgeObj.depth; k++) {
         depthToNodes.push([]);
@@ -325,10 +329,23 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       } else {
         // Existing node
         if (edgeObj.depth > node.depth) {
-          var oldIndex = depthToNodes[node.depth].indexOf(node);
-          depthToNodes[node.depth].splice(oldIndex, 1);
-          node.depth = edgeObj.depth;
-          depthToNodes[node.depth].push(node);
+          function moveNodeToDepth(node, newDepth) {
+            console.debug("Moving " + node.name + " from depth " + node.depth + " to " + newDepth);
+            var oldIndex = depthToNodes[node.depth].indexOf(node);
+            depthToNodes[node.depth].splice(oldIndex, 1);
+            node.depth = newDepth;
+            if (!depthToNodes[node.depth]) {
+              depthToNodes[node.depth] = [];
+            }
+            depthToNodes[node.depth].push(node);
+            if (node.children) {
+              // Move children down depth
+              node.children.forEach(function(c) {
+                moveNodeToDepth(c, node.depth + 1);
+              })
+            }
+          }
+          moveNodeToDepth(node, edgeObj.depth);
         }
       }
       if (depthToNodes[node.depth].length > maxRowLength) {
@@ -509,6 +526,23 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
     //
     var self = this;
     var force = d3.layout.force().nodes(dagVisualModel.nodes).links(dagVisualModel.links).start();
+    var nodeDragData = {
+      nodeRelativeX : 0,
+      nodeRelativeY : 0
+    };
+    var nodeDrag = d3.behavior.drag().on('dragstart', function(node){
+      d3.event.sourceEvent.stopPropagation();
+      var rc = d3.mouse(this);
+      nodeDragData.nodeRelativeX = rc[0];
+      nodeDragData.nodeRelativeY = rc[1];
+    }).on('drag', function(node){
+      var nx = d3.event.x - nodeDragData.nodeRelativeX;
+      var ny = d3.event.y - nodeDragData.nodeRelativeY;
+      self.dragVertex(d3.select(this), node, [nx, ny], diagonal);
+    }).on('dragend', function(){
+      nodeDragData.nodeRelativeX = 0;
+      nodeDragData.nodeRelativeY = 0;
+    });
     // Create Links
     var diagonal = d3.svg.diagonal().source(function(d) {
       return {
@@ -521,7 +555,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
         y : d.target.incomingY - 12
       }
     });
-    var link = svgLayer.selectAll(".link").data(dagVisualModel.links).enter().append("g").attr("class", "link").attr("marker-end", "url(#arrow)");
+    var link = svgLayer.selectAll(".link-g").data(dagVisualModel.links).enter().append("g").attr("class", "link-g").attr("marker-end", "url(#arrow)");
     link.append("path").attr("class", function(l) {
       var classes = "link svg-tooltip ";
       switch (l.edgeType) {
@@ -540,7 +574,7 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
       return Em.I18n.t("jobs.hive.tez.edge." + lower);
     });
     // Create Nodes
-    var node = svgLayer.selectAll(".node").data(dagVisualModel.nodes).enter().append("g").attr("class", "node");
+    var node = svgLayer.selectAll(".node").data(dagVisualModel.nodes).enter().append("g").attr("class", "node").call(nodeDrag);
     node.append("rect").attr("class", "background").attr("width", function(n) {
       return n.width;
     }).attr("height", function(n) {
@@ -668,6 +702,25 @@ App.MainHiveJobDetailsTezDagView = Em.View.extend({
     }
   },
 
+  dragVertex: function(d3Vertex, node, newPosition, diagonal){
+    console.debug("Dragging vertex [", node.name, "] to (", newPosition[0], ",", newPosition[1], ")");
+    // Move vertex
+    node.x = newPosition[0];
+    node.y = newPosition[1];
+    node.incomingX = newPosition[0] + (node.width/2);
+    node.incomingY = newPosition[1];
+    node.outgoingX = newPosition[0] + (node.width/2);
+    node.outgoingY = newPosition[1] + node.height;
+    d3Vertex.attr('transform', 'translate(' + newPosition[0] + ',' + newPosition[1] + ')');
+    // Move links
+    d3.selectAll('.link').filter(function(l) {
+      if (l && (l.source === node || l.target === node)) {
+        return this
+      }
+      return null;
+    }).attr('d', diagonal);
+  },
+
   /**
    * Refreshes UI of the Tez graph with latest values
    */