AMBARI-4466. Jobs: Create Mappers for Hive and Tez query data. (srimanth)

ambari-web/app/assets/data/jobs/hive-queries.json

@@ -0,0 +1,92 @@
+{
+  "entities": [
+    {
+      "otherinfo": {},
+      "primaryfilters": {
+        "user": "hiveuser1"
+      },
+      "events": [
+        {
+          "ts": 1390516285963,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390516278010,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "keyts": 1390516278010,
+      "relatedentities": {},
+      "entity": "HIVE-Q3",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "otherinfo": {},
+      "primaryfilters": {
+        "user": "hiveuser2"
+      },
+      "events": [
+        {
+          "ts": 1390516133110,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390516133110,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "keyts": 1390516133110,
+      "relatedentities": {},
+      "entity": "HIVE-Q2",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "otherinfo": {},
+      "primaryfilters": {
+        "user": "hiveuser1"
+      },
+      "events": [
+        {
+          "ts": 1390515297205,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390515264503,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "keyts": 1390515264503,
+      "relatedentities": {},
+      "entity": "HIVE-Q1",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "otherinfo": {},
+      "primaryfilters": {
+        "user": "hiveuser2"
+      },
+      "events": [
+        {
+          "ts": 1390516418183,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1389809461915,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "keyts": 1389809461915,
+      "relatedentities": {},
+      "entity": "HIVE-Q4",
+      "entitytype": "HIVE_QUERY_ID"
+    }
+  ]
+}

ambari-web/app/assets/data/jobs/hive-query-2.json

@@ -0,0 +1,223 @@
+{
+  "relatedentities": {},
+  "primaryfilters": {
+    "user": "hiveuser2"
+  },
+  "secondaryfilters": {},
+  "otherinfo": {
+    "startTime": 1390516133110,
+    "query": {
+      "queryText": "INSERT OVERWRITE TABLE dest_j1 SELECT x.key, z.value, y.value FROM src1 x JOIN src y ON (x.key = y.key) JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)",
+      "queryPlan": {
+        "STAGE PLANS": {
+          "Stage-8": {
+            "Conditional Operator": {}
+          },
+          "Stage-2": {
+            "Dependency Collection": {}
+          },
+          "Stage-1": {
+            "Tez": {
+              "DagName": "HIVE-Q2:1",
+              "Vertices:": {
+                "Map 1": {
+                  "Map Operator Tree:": [
+                    {
+                      "TableScan": {
+                        "alias:": "z",
+                        "children": {
+                          "Reduce Output Operator": {
+                            "Map-reduce partition columns:": "value (type: string)",
+                            "sort order:": "+",
+                            "value expressions:": "value (type: string)",
+                            "Statistics:": "Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE",
+                            "key expressions:": "value (type: string)"
+                          }
+                        },
+                        "Statistics:": "Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE"
+                      }
+                    }
+                  ]
+                },
+                "Map 2": {
+                  "Map Operator Tree:": [
+                    {
+                      "TableScan": {
+                        "alias:": "y",
+                        "children": {
+                          "Map Join Operator": {
+                            "keys:": {
+                              "0": "key (type: string)",
+                              "1": "key (type: string)"
+                            },
+                            "outputColumnNames:": [
+                              "_col0",
+                              "_col1",
+                              "_col5"
+                            ],
+                            "children": {
+                              "Map Join Operator": {
+                                "keys:": {
+                                  "0": "_col1 (type: string)",
+                                  "1": "value (type: string)"
+                                },
+                                "outputColumnNames:": [
+                                  "_col1",
+                                  "_col4",
+                                  "_col9"
+                                ],
+                                "children": {
+                                  "Select Operator": {
+                                    "expressions:": "_col4 (type: string), _col9 (type: string), _col1 (type: string)",
+                                    "outputColumnNames:": [
+                                      "_col0",
+                                      "_col1",
+                                      "_col2"
+                                    ],
+                                    "children": {
+                                      "File Output Operator": {
+                                        "Statistics:": "Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE",
+                                        "compressed:": "false",
+                                        "table:": {
+                                          "serde:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
+                                          "name:": "default.dest_j1",
+                                          "input format:": "org.apache.hadoop.mapred.TextInputFormat",
+                                          "output format:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+                                        }
+                                      }
+                                    },
+                                    "Statistics:": "Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE"
+                                  }
+                                },
+                                "Statistics:": "Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE",
+                                "condition map:": [
+                                  {
+                                    "": "Inner Join 0 to 1"
+                                  }
+                                ],
+                                "condition expressions:": {
+                                  "0": "{_col5} {_col0}",
+                                  "1": "{value}"
+                                }
+                              }
+                            },
+                            "Statistics:": "Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE",
+                            "condition map:": [
+                              {
+                                "": "Inner Join 0 to 1"
+                              }
+                            ],
+                            "condition expressions:": {
+                              "0": "{key} {value}",
+                              "1": "{value}"
+                            }
+                          }
+                        },
+                        "Statistics:": "Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE"
+                      }
+                    }
+                  ]
+                },
+                "Map 3": {
+                  "Map Operator Tree:": [
+                    {
+                      "TableScan": {
+                        "alias:": "x",
+                        "children": {
+                          "Reduce Output Operator": {
+                            "Map-reduce partition columns:": "key (type: string)",
+                            "sort order:": "+",
+                            "value expressions:": "key (type: string), value (type: string)",
+                            "Statistics:": "Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE",
+                            "key expressions:": "key (type: string)"
+                          }
+                        },
+                        "Statistics:": "Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE"
+                      }
+                    }
+                  ]
+                }
+              },
+              "Edges:": {
+                "Map 2": [
+                  {
+                    "parent": "Map 1",
+                    "type": "BROADCAST_EDGE"
+                  },
+                  {
+                    "parent": "Map 3",
+                    "type": "BROADCAST_EDGE"
+                  }
+                ]
+              }
+            }
+          },
+          "Stage-0": {
+            "Move Operator": {
+              "tables:": {
+                "replace:": "true"
+              }
+            }
+          },
+          "Stage-5": {
+            "Move Operator": {
+              "files:": {
+                "destination:": "hdfs://localhost:19000/tmp/hive-Hitesh/hive_2014-01-23_14-22-57_727_4019734356848899035-1/-ext-10000",
+                "hdfs directory:": "true"
+              }
+            }
+          },
+          "Stage-4": {
+            "Tez": {
+              "Vertices:": {
+                "Merge": {
+                  "Map Operator Tree:": [
+                    {
+                      "TableScan": {
+                        "children": {
+                          "File Output Operator": {
+                            "compressed:": "false"
+                          }
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+            }
+          },
+          "Stage-3": {
+            "Stats-Aggr Operator": {}
+          }
+        },
+        "STAGE DEPENDENCIES": {
+          "Stage-8": {
+            "DEPENDENT STAGES": "Stage-1",
+            "CONDITIONAL CHILD TASKS": "Stage-5, Stage-4, Stage-6"
+          },
+          "Stage-7": {
+            "DEPENDENT STAGES": "Stage-6"
+          },
+          "Stage-2": {
+            "DEPENDENT STAGES": "Stage-5, Stage-4, Stage-7"
+          },
+          "Stage-1": {
+            "ROOT STAGE": "TRUE"
+          },
+          "Stage-0": {
+            "DEPENDENT STAGES": "Stage-2"
+          },
+          "Stage-6": {},
+          "Stage-5": {},
+          "Stage-4": {},
+          "Stage-3": {
+            "DEPENDENT STAGES": "Stage-0"
+          }
+        }
+      }
+    },
+    "endTime": 1390516133110
+  },
+  "entity": "HIVE-Q2",
+  "entitytype": "HIVE_QUERY_ID"
+}

ambari-web/app/assets/data/jobs/hive-query-events.json

@@ -0,0 +1,68 @@
+{
+  "events": [
+    {
+      "events": [
+        {
+          "ts": 1390515297205,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390515264503,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "entity": "HIVE-Q1",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "events": [
+        {
+          "ts": 1390516133110,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390516133110,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "entity": "HIVE-Q2",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "events": [
+        {
+          "ts": 1390516285963,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1390516278010,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "entity": "HIVE-Q3",
+      "entitytype": "HIVE_QUERY_ID"
+    },
+    {
+      "events": [
+        {
+          "ts": 1390516418183,
+          "eventtype": "QUERY_COMPLETED",
+          "eventinfo": {}
+        },
+        {
+          "ts": 1389809461915,
+          "eventtype": "QUERY_SUBMITTED",
+          "eventinfo": {}
+        }
+      ],
+      "entity": "HIVE-Q4",
+      "entitytype": "HIVE_QUERY_ID"
+    }
+  ]
+}

ambari-web/app/assets/data/jobs/tezDag-name-to-id.json

@@ -0,0 +1,13 @@
+{
+  "entities": [
+    {
+      "ts": 1390516133120,
+      "info": {
+        "dagName": "HIVE-Q2:1"
+      },
+      "entity": "dag_1390516007863_0001_1",
+      "eventtype": "DAG_SUBMITTED"
+    }
+  ],
+  "entitytype": "TEZ_DAG_ID"
+}

ambari-web/app/assets/data/jobs/tezDag.json

@@ -0,0 +1,172 @@
+{
+  "relatedentities": {
+    "TEZ_VERTEX_ID": [
+      "vertex_1390516007863_0001_1_00",
+      "vertex_1390516007863_0001_1_01",
+      "vertex_1390516007863_0001_1_02"
+    ]
+  },
+  "primaryfilters": {},
+  "secondaryfilters": {},
+  "otherinfo": {
+    "startTime": 1390516133217,
+    "status": "SUCCEEDED",
+    "timeTaken": 9408,
+    "dagPlan": {
+      "dagName": "HIVE-Q2:1",
+      "edges": [
+        {
+          "edgeId": "1167505869",
+          "inputVertexName": "Map 3",
+          "outputVertexName": "Map 2",
+          "dataMovementType": "BROADCAST",
+          "dataSourceType": "PERSISTED",
+          "schedulingType": "SEQUENTIAL",
+          "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+          "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+        },
+        {
+          "edgeId": "360567249",
+          "inputVertexName": "Map 1",
+          "outputVertexName": "Map 2",
+          "dataMovementType": "BROADCAST",
+          "dataSourceType": "PERSISTED",
+          "schedulingType": "SEQUENTIAL",
+          "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+          "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+        }
+      ],
+      "vertices": [
+        {
+          "vertexName": "Map 1",
+          "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+          "outEdgeIds": "360567249",
+          "additionalInputs": {
+            "name": "z",
+            "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+            "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+          }
+        },
+        {
+          "vertexName": "Map 2",
+          "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+          "inEdgeIds": [
+            "1167505869",
+            "360567249"
+          ],
+          "additionalInputs": {
+            "name": "y",
+            "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+            "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+          },
+          "additionalOutputs": {
+            "name": "out_Map 2",
+            "class": "org.apache.tez.mapreduce.output.MROutput"
+          }
+        },
+        {
+          "vertexName": "Map 3",
+          "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+          "outEdgeIds": "1167505869",
+          "additionalInputs": {
+            "name": "x",
+            "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+            "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+          }
+        }
+      ]
+    },
+    "endTime": 1390516142625,
+    "counters": {
+      "counterGroups": [
+        {
+          "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
+          "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
+          "counters": [
+            {
+              "counterName": "TOTAL_LAUNCHED_TASKS",
+              "counterDisplayName": "TOTAL_LAUNCHED_TASKS",
+              "counterValue": 3
+            },
+            {
+              "counterName": "DATA_LOCAL_TASKS",
+              "counterDisplayName": "DATA_LOCAL_TASKS",
+              "counterValue": 3
+            }
+          ]
+        },
+        {
+          "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
+          "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
+          "counters": [
+            {
+              "counterName": "MAP_INPUT_RECORDS",
+              "counterDisplayName": "MAP_INPUT_RECORDS",
+              "counterValue": 1025
+            },
+            {
+              "counterName": "MAP_OUTPUT_RECORDS",
+              "counterDisplayName": "MAP_OUTPUT_RECORDS",
+              "counterValue": 0
+            }
+          ]
+        },
+        {
+          "counterGroupName": "HIVE",
+          "counterGroupDisplayName": "HIVE",
+          "counters": {
+            "counterName": "CREATED_FILES",
+            "counterDisplayName": "CREATED_FILES",
+            "counterValue": 1
+          }
+        },
+        {
+          "counterGroupName": "default.dest_j1/",
+          "counterGroupDisplayName": "default.dest_j1/",
+          "counters": [
+            {
+              "counterName": "numRows",
+              "counterDisplayName": "numRows",
+              "counterValue": 85
+            },
+            {
+              "counterName": "rawDataSize",
+              "counterDisplayName": "rawDataSize",
+              "counterValue": 1600
+            }
+          ]
+        },
+        {
+          "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counters": {
+            "counterName": "DESERIALIZE_ERRORS",
+            "counterDisplayName": "DESERIALIZE_ERRORS",
+            "counterValue": 0
+          }
+        },
+        {
+          "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
+          "counterGroupDisplayName": "File Input Format Counters ",
+          "counters": {
+            "counterName": "BYTES_READ",
+            "counterDisplayName": "Bytes Read",
+            "counterValue": 0
+          }
+        },
+        {
+          "counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+          "counterGroupDisplayName": "File Output Format Counters ",
+          "counters": {
+            "counterName": "BYTES_WRITTEN",
+            "counterDisplayName": "Bytes Written",
+            "counterValue": 0
+          }
+        }
+      ]
+    },
+    "diagnostics": ""
+  },
+  "entity": "dag_1390516007863_0001_1",
+  "entitytype": "TEZ_DAG_ID"
+}

ambari-web/app/assets/data/jobs/tezDagVertex.json

@@ -0,0 +1,64 @@
+{
+  "relatedentities": {
+    "TEZ_TASK_ID": [
+      "task_1390516007863_0001_1_00_000000"
+    ]
+  },
+  "primaryfilters": {},
+  "secondaryfilters": {},
+  "otherinfo": {
+    "startTime": 1390516133853,
+    "initTime": 1390516133853,
+    "numTasks": 1,
+    "status": "SUCCEEDED",
+    "vertexName": "Map 1",
+    "timeTaken": 8110,
+    "processorClassName": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+    "endTime": 1390516141963,
+    "counters": {
+      "counterGroups": [
+        {
+          "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
+          "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
+          "counters": {
+            "counterName": "DATA_LOCAL_TASKS",
+            "counterDisplayName": "DATA_LOCAL_TASKS",
+            "counterValue": 1
+          }
+        },
+        {
+          "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
+          "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
+          "counters": {
+            "counterName": "MAP_INPUT_RECORDS",
+            "counterDisplayName": "MAP_INPUT_RECORDS",
+            "counterValue": 500
+          }
+        },
+        {
+          "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+          "counters": {
+            "counterName": "DESERIALIZE_ERRORS",
+            "counterDisplayName": "DESERIALIZE_ERRORS",
+            "counterValue": 0
+          }
+        },
+        {
+          "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
+          "counterGroupDisplayName": "File Input Format Counters ",
+          "counters": {
+            "counterName": "BYTES_READ",
+            "counterDisplayName": "Bytes Read",
+            "counterValue": 0
+          }
+        }
+      ]
+    },
+    "diagnostics": "",
+    "initRequestedTime": 1390516133331,
+    "startRequestedTime": 1390516133415
+  },
+  "entity": "vertex_1390516007863_0001_1_00",
+  "entitytype": "TEZ_VERTEX_ID"
+}

ambari-web/app/controllers.js

@@ -118,6 +118,7 @@ require('controllers/main/mirroring/targetClusterController');
 require('controllers/main/mirroring/testConnection_controller');
 require('controllers/main/mirroring/testConnectionResults_controller');
 require('controllers/main/mirroring/manage_clusters_controller');
+require('controllers/main/jobs_controller');
 require('controllers/wizard/slave_component_groups_controller');
 require('controllers/wizard/step0_controller');
 require('controllers/wizard/step1_controller');

ambari-web/app/controllers/main/jobs_controller.js

@@ -18,7 +18,7 @@
 
 var App = require('app');
 
-App.MainJobsController = App.MainAppsController.extend({
+App.MainJobsController = Em.ArrayController.extend({
 
   name:'mainJobsController',
 
@@ -26,6 +26,11 @@ App.MainJobsController = App.MainAppsController.extend({
 
   loaded : false,
   loading : false,
+  /**
+   * The number of jobs to be shown by last submitted time.
+   */
+  jobsLimit : -1,
+
 
   clearFilters: function () {
     var obj=this.get("filterObject");
@@ -60,6 +65,30 @@ App.MainJobsController = App.MainAppsController.extend({
       { name: Em.I18n.t('jobs.column.end.time'), index: 3 },
       { name: Em.I18n.t('jobs.column.duration'), index: 4 }
     ]
-  })
+  }),
+
+  loadJobs : function() {
+    var self = this;
+    var jobsLimit = this.get('jobsLimit');
+    var yarnService = App.YARNService.find().objectAt(0);
+    if (yarnService != null) {
+      this.set('loading', true);
+      var historyServerHostName = yarnService.get('resourceManagerNode.hostName')
+      var hiveQueriesUrl = App.testMode ? "/data/jobs/hive-queries.json" : App.apiPrefix + "/proxy?url=http://" + historyServerHostName
+          + ":8188/ws/v1/apptimeline/HIVE_QUERY_ID?fields=events,primaryfilters";
+      if (jobsLimit > 0) {
+        hiveQueriesUrl += ("?limit=" + jobsLimit);
+      }
+      App.HttpClient.get(hiveQueriesUrl, App.hiveJobsMapper, {
+        complete : function(jqXHR, textStatus) {
+          self.set('loading', false);
+          self.set('loaded', true);
+        }
+      });
+    }
+  },
 
+  refreshLoadedJobs : function() {
+    this.loadJobs();
+  }.observes('jobsLimit', 'App.router.clusterController.isLoaded')
 })

ambari-web/app/initialize.js

@@ -48,6 +48,8 @@ require('mappers/service_metrics_mapper');
 require('mappers/target_cluster_mapper');
 require('mappers/dataset_mapper');
 require('mappers/component_config_mapper');
+require('mappers/jobs/hive_jobs_mapper');
+require('mappers/jobs/hive_job_mapper');
 require('utils/http_client');
 require('utils/host_progress_popup');
 

ambari-web/app/mappers/jobs/hive_job_mapper.js

@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+
+/**
+ * Maps a Hive job response from server into an existing Hive Job.
+ *
+ * This will only update an existing Hive Job and not create a new one. It will
+ * populate all fields (stages, Tez DAG, etc.) except runtime information like
+ * <ul>
+ * <li>tezDag.id
+ * <li>tezDag.vertices.state
+ * <li>tezDag.vertices.startTime
+ * <li>tezDag.vertices.endTime
+ * <li>tezDag.vertices.tasksCount
+ * <li>tezDag.vertices.file*
+ * <li>tezDag.vertices.hdfs*
+ * <li>tezDag.vertices.recordReadCount
+ * <li>tezDag.vertices.recordWriteCount
+ * </ul>
+ */
+App.hiveJobMapper = App.QuickDataMapper.create({
+  model : App.HiveJob,
+  map : function(json) {
+    var model = this.get('model');
+    if (!model) {
+      return;
+    }
+    var hiveJob = {}
+    if (json && json.entity) {
+      var hiveJob = {};
+      hiveJob.id = json.entity;
+      hiveJob.startTime = json.otherinfo.startTime;
+      hiveJob.endTime = json.otherinfo.endTime;
+      hiveJob.queryText = json.otherinfo.query.queryText;
+      hiveJob.stages = [];
+      var stagePlans = json.otherinfo.query.queryPlan["STAGE PLANS"];
+      for ( var stage in stagePlans) {
+        hiveJob.stages.push(stage);
+        var stageValue = stagePlans[stage];
+        if (stageValue.Tez != null && hiveJob.tezDag == null) {
+          var dagName = stageValue.Tez.DagName;
+          // Vertices
+          var vertices = [];
+          var vertexIds = [];
+          for ( var vertexName in stageValue.Tez["Vertices:"]) {
+            var vertex = stageValue.Tez["Vertices:"][vertexName];
+            var vertexObj = {
+              id : dagName + "/" + vertexName,
+              name : vertexName,
+            };
+            vertexIds.push(vertexObj.id);
+            var operatorExtractor = function(obj) {
+              var ops = [];
+              if ($.isArray(obj)) {
+                obj.forEach(function(o) {
+                  ops = ops.concat(operatorExtractor(o));
+                });
+              } else {
+                for ( var key in obj) {
+                  ops.push(key);
+                  if (obj[key].children != null) {
+                    ops = ops.concat(operatorExtractor(obj[key].children));
+                  }
+                }
+              }
+              return ops;
+            }
+            if (vertex["Map Operator Tree:"] != null) {
+              vertexObj.isMap = true;
+              vertexObj.operations = operatorExtractor(vertex["Map Operator Tree:"]);
+              vertexObj.operationPlan = JSON.stringify(vertex["Map Operator Tree:"]);
+            } else if (vertex["Reduce Operator Tree:"] != null) {
+              vertexObj.isMap = false;
+              vertexObj.operations = operatorExtractor(vertex["Reduce Operator Tree:"]);
+              vertexObj.operationPlan = JSON.stringify(vertex["Reduce Operator Tree:"]);
+            }
+            vertices.push(vertexObj);
+          }
+          // Edges
+          var edges = [];
+          var edgeIds = [];
+          for ( var childVertex in stageValue.Tez["Edges:"]) {
+            stageValue.Tez["Edges:"][childVertex].forEach(function(e) {
+              var parentVertex = e.parent;
+              var edgeObj = {
+                id : dagName + "/" + parentVertex + "-" + childVertex,
+                fromVertex : dagName + "/" + parentVertex,
+                toVertex : dagName + "/" + childVertex
+              };
+              edgeIds.push(edgeObj.id);
+              switch (e.type) {
+              case "BROADCAST_EDGE":
+                edgeObj.edgeType = App.TezDagVertexType.BROADCAST;
+                break;
+              case "SIMPLE_EDGE":
+                edgeObj.edgeType = App.TezDagVertexType.SCATTER_GATHER;
+                break;
+              default:
+                break;
+              }
+              edges.push(edgeObj);
+            });
+          }
+          // Create records
+          var tezDag = {
+            id : dagName,
+            name : dagName,
+            stage : stage,
+            vertices : vertexIds,
+            edges : edgeIds
+          }
+          App.store.loadMany(App.TezDagVertex, vertices);
+          App.store.loadMany(App.TezDagEdge, edges);
+          App.store.load(App.TezDag, tezDag);
+          hiveJob.tezDag = tezDag.id;
+        }
+      }
+      var hiveJobRecord = App.HiveJob.find(hiveJob.id);
+      if (hiveJobRecord != null) {
+        hiveJobRecord.set('queryText', hiveJob.queryText);
+        hiveJobRecord.set('stages', hiveJob.stages);
+        hiveJobRecord.set('startTime', hiveJob.startTime);
+        hiveJobRecord.set('endTime', hiveJob.endTime);
+        hiveJobRecord.set('tezDag', App.TezDag.find(hiveJob.tezDag));
+      }
+    }
+  },
+  config : {}
+});

ambari-web/app/mappers/jobs/hive_jobs_mapper.js

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+
+App.hiveJobsMapper = App.QuickDataMapper.create({
+  model : App.HiveJob,
+  map : function(json) {
+    var model = this.get('model');
+    if (!model) {
+      return;
+    }
+    var hiveJobs = []
+    if (json && json.entities) {
+      var currentEntityMap = {}
+      json.entities.forEach(function(entity) {
+        currentEntityMap[entity.entity] = entity.entity;
+        var hiveJob = {
+          id : entity.entity,
+          name : entity.entity,
+          user : entity.primaryfilters.user
+        }
+        if (entity.events != null) {
+          entity.events.forEach(function(event) {
+            switch (event.eventtype) {
+            case "QUERY_SUBMITTED":
+              hiveJob.start_time = event.ts;
+              break;
+            case "QUERY_COMPLETED":
+              hiveJob.end_time = event.ts;
+              break;
+            default:
+              break;
+            }
+          });
+        }
+        hiveJobs.push(hiveJob);
+      });
+      // Delete IDs not seen from server
+      var currentModels = model.find();
+      currentModels.forEach(function(m) {
+        if (!currentEntityMap[m.get('id')]) {
+          this.deleteRecord(m);
+        }
+      }, this);
+    }
+    App.store.loadMany(model, hiveJobs);
+    App.router.get('mainJobsController').set('content', App.HiveJob.find().toArray());
+  },
+  config : {}
+});

ambari-web/app/models/jobs/job.js

@@ -30,16 +30,39 @@ App.AbstractJob = DS.Model.extend({
   startTime : DS.attr('number'),
   endTime : DS.attr('number'),
 
+  startTimeDisplay : function() {
+    var startTime = this.get('startTime');
+    if (startTime > 0) {
+      return dateUtils.dateFormat(startTime);
+    } else {
+      return "";
+    }
+  }.property('startTime'),
+
+  endTimeDisplay : function() {
+    var endTime = this.get('endTime');
+    if (endTime > 0) {
+      return dateUtils.dateFormat(endTime);
+    } else {
+      return "";
+    }
+  }.property('endTime'),
+
   /**
    * Provides the duration of this job. If the job has not started, duration
    * will be given as 0. If the job has not ended, duration will be till now.
-   *
+   * 
    * @return {Number} Duration in milliseconds.
    */
   duration : function() {
     return dateUtils.duration(this.get('startTime'), this.get('endTime'))
   }.property('startTime', 'endTime'),
 
+  durationDisplay : function() {
+    var duration = this.get('duration');
+    return dateUtils.timingFormat(duration, true);
+  }.property('duration'),
+
   /**
    * Type of this job. Should be one of constants defined in App.JobType
    */

ambari-web/app/models/jobs/tez_dag.js

@@ -20,14 +20,19 @@ var dateUtils = require('utils/date');
 
 App.TezDag = DS.Model.extend({
   id : DS.attr('string'),
+  /**
+   * When DAG is actually running on server, a unique ID is assigned.
+   */
+  insanceId : DS.attr('string'),
   name : DS.attr('string'),
   stage : DS.attr('string'),
   vertices : DS.hasMany('App.TezDagVertex'),
-  edges: DS.hasMany('App.TezDagEdge')
+  edges : DS.hasMany('App.TezDagEdge')
 });
 
 App.TezDagEdge = DS.Model.extend({
   id : DS.attr('string'),
+  insanceId : DS.attr('string'),
   fromVertex : DS.belongsTo('App.TezDagVertex'),
   toVertex : DS.belongsTo('App.TezDagVertex'),
   /**
@@ -39,6 +44,10 @@ App.TezDagEdge = DS.Model.extend({
 
 App.TezDagVertex = DS.Model.extend({
   id : DS.attr('string'),
+  /**
+   * When DAG vertex is actually running on server, a unique ID is assigned.
+   */
+  insanceId : DS.attr('string'),
   name : DS.attr('string'),
 
   /**
@@ -55,12 +64,12 @@ App.TezDagVertex = DS.Model.extend({
   /**
    * A vertex can have multiple incoming edges.
    */
-  incomingEdges: DS.hasMany('App.TezDagEdge'),
+  incomingEdges : DS.hasMany('App.TezDagEdge'),
 
   /**
    * This vertex can have multiple outgoing edges.
    */
-  outgoingEdges: DS.hasMany('App.TezDagEdge'),
+  outgoingEdges : DS.hasMany('App.TezDagEdge'),
 
   startTime : DS.attr('number'),
   endTime : DS.attr('number'),
@@ -89,7 +98,7 @@ App.TezDagVertex = DS.Model.extend({
   operationPlan : DS.attr('string'),
 
   /**
-   * Number of tasks in this vertex
+   * Number of actual Map/Reduce tasks in this vertex
    */
   tasksCount : DS.attr('number'),
 

ambari-web/app/routes/main.js

@@ -114,8 +114,14 @@ module.exports = Em.Route.extend({
   jobs: Em.Route.extend({
     route: '/jobs',
     connectOutlets: function (router) {
-      router.get('mainJobsController').loadRuns();
-      router.get('mainController').connectOutlet('mainJobs');
+      if (!App.get('isHadoop2Stack')) {
+        Em.run.next(function () {
+          router.transitionTo('main.dashboard');
+        });
+      } else {
+        router.get('mainJobsController').loadJobs();
+        router.get('mainController').connectOutlet('mainJobs');
+      }
     }
   }),
 

ambari-web/app/templates/main/jobs.hbs

@@ -51,8 +51,24 @@
                 <td class="no-data" {{bindAttr colspan="controller.columnsName.content.length"}}>{{t apps.filters.nothingToShow}}</td>
             </tr>
         {{else}}
-          {{#each run in content}}
-            {{view view.containerRow runBinding="run" currentViewBinding="view.appTableRow"}}
+          {{#each job in content}}
+            <tr>
+              <td width="40%">
+                {{job.name}}
+              </td>
+              <td>
+                {{job.user}}
+              </td>
+              <td>
+                {{job.startTimeDisplay}}
+              </td>
+              <td>
+                {{job.endTimeDisplay}}
+              </td>
+              <td>
+                {{job.durationDisplay}}
+              </td>
+            </tr>
           {{/each}}
         {{/if}}
         </tbody>

ambari-web/app/utils/ajax.js

@@ -1510,6 +1510,18 @@ var urls = {
         data: data.dataset
       }
     }
+  },
+  'jobs.tezDag.NametoID': {
+    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_DAG_ID?primaryFilter=dagName:{tezDagName}',
+    'mock': '/data/jobs/tezDag-name-to-id.json'
+  },
+  'jobs.tezDag.tezDagId': {
+    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_DAG_ID/{tezDagId}',
+    'mock': '/data/jobs/tezDag.json'
+  },
+  'jobs.tezDag.tezDagVertexId': {
+    'real': '/proxy?url=http://{historyServerHostName}:8188/ws/v1/apptimeline/TEZ_VERTEX_ID/{tezDagVertexId}',
+    'mock': '/data/jobs/tezDagVertex.json'
   }
 };
 /**

ambari-web/app/utils/jobs.js

@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+module.exports = {
+
+  /**
+   * Refreshes the latest information for a given job
+   *
+   * @param {App.AbstractJob}
+   *          job
+   */
+  refreshJobDetails : function(job) {
+    if (job) {
+      switch (job.get('jobType')) {
+      case App.JobType.HIVE:
+        this.refreshHiveJobDetails(job);
+        break;
+      default:
+        break;
+      }
+    }
+  },
+
+  /**
+   * Refreshes latest information of a Hive Job.
+   *
+   * @param {App.HiveJob}
+   *          hiveJob
+   */
+  refreshHiveJobDetails : function(hiveJob) {
+    var self = this;
+    // TODO - to be changed to history server when implemented in stack.
+    var historyServerHostName = App.YARNService.find().objectAt(0).get('resourceManagerNode.hostName')
+    var hiveJobId = hiveJob.get('id');
+    // First refresh query
+    var hiveQueriesUrl = App.testMode ? "/data/jobs/hive-query-2.json" :
+    App.apiPrefix + "/proxy?url=http://"+historyServerHostName+":8188/ws/v1/apptimeline/HIVE_QUERY_ID/" + hiveJob.get('id');
+    App.HttpClient.get(hiveQueriesUrl, App.hiveJobMapper, {
+      complete : function(jqXHR, textStatus) {
+        // Now get the Tez DAG ID from the DAG name
+        var hiveRecord = App.HiveJob.find(hiveJobId);
+        var tezDagName = hiveRecord.get('tezDag.name');
+        if (tezDagName != null) {
+          var sender = {
+            dagNameToIdSuccess : function(data) {
+              if (data && data.entities && data.entities.length > 0) {
+                var dagId = data.entities[0].entity;
+                App.TezDag.find(tezDagName).set('instanceId', dagId);
+                self.refreshTezDagDetails(tezDagName);
+              }
+            },
+            dagNameToIdError : function(jqXHR, url, method, showStatus) {
+              App.ajax.defaultErrorHandler(jqXHR, url, method, showStatus);
+            }
+          }
+          App.ajax.send({
+            name : 'jobs.tezDag.NametoID',
+            sender : sender,
+            data : {
+              historyServerHostName : historyServerHostName,
+              tezDagName : tezDagName
+            },
+            success : 'dagNameToIdSuccess',
+            error : 'dagNameToIdError'
+          });
+        } else {
+          // TODO
+        }
+
+      }
+    });
+  },
+
+  /**
+   * Refreshes runtime information of a Tez DAG based on events generated. The
+   * instance ID of the Tez DAG should be set.
+   *
+   * @param {string}
+   *          tezDagId ID of the Tez DAG. Example: 'HIVE-Q2:1'
+   */
+  refreshTezDagDetails : function(tezDagId) {
+    var self = this;
+    var historyServerHostName = App.YARNService.find().objectAt(0).get('resourceManagerNode.hostName');
+    var tezDag = App.TezDag.find(tezDagId);
+    if (tezDag) {
+      var tezDagInstanceId = tezDag.get('instanceId');
+      var sender = {
+        loadTezDagSuccess : function(data) {
+          if (data && data.relatedentities && data.relatedentities.TEZ_VERTEX_ID != null) {
+            data.relatedentities.TEZ_VERTEX_ID.forEach(function(v) {
+              self.refreshTezDagVertex(tezDagId, v);
+            });
+          }
+        },
+        loadTezDagError : function(jqXHR, url, method, showStatus) {
+          App.ajax.defaultErrorHandler(jqXHR, url, method, showStatus);
+        }
+      }
+      App.ajax.send({
+        name : 'jobs.tezDag.tezDagId',
+        sender : sender,
+        data : {
+          historyServerHostName : historyServerHostName,
+          tezDagId : tezDagInstanceId
+        },
+        success : 'loadTezDagSuccess',
+        error : 'loadTezDagError'
+      });
+    }
+  },
+
+  /**
+   * Refreshes runtime information of the given vertex.
+   *
+   * @param {string}
+   *          tezDagId ID of the Tez DAG. Exmaple: 'HIVE-Q2:1'
+   * @param {string}
+   *          tezVertexInstanceID Instance ID of the vertex to refresh. Example
+   *          'vertex_1390516007863_0001_1_00'
+   */
+  refreshTezDagVertex : function(tezDagId, tezVertexInstanceId) {
+    var historyServerHostName = App.YARNService.find().objectAt(0).get('resourceManagerNode.hostName');
+    var sender = {
+      loadTezDagVertexSuccess : function(data) {
+        if (data && data.otherinfo) {
+          var vertexRecord = App.TezDagVertex.find(tezDagId + "/" + data.otherinfo.vertexName);
+          if (vertexRecord != null) {
+            vertexRecord.set('startTime', data.otherinfo.startTime);
+            vertexRecord.set('endTime', data.otherinfo.startTime + +data.otherinfo.timeTaken);
+            vertexRecord.set('tasksCount', data.otherinfo.numTasks);
+            vertexRecord.set('state', data.otherinfo.status);
+            // TODO Need additional vertex metrics
+          }
+        }
+      },
+      loadTezDagVertexError : function(jqXHR, url, method, showStatus) {
+        App.ajax.defaultErrorHandler(jqXHR, url, method, showStatus);
+      }
+    }
+    App.ajax.send({
+      name : 'jobs.tezDag.tezDagVertexId',
+      sender : sender,
+      data : {
+        historyServerHostName : historyServerHostName,
+        tezDagVertexId : tezVertexInstanceId
+      },
+      success : 'loadTezDagVertexSuccess',
+      error : 'loadTezDagVertexError'
+    });
+  }
+};

ambari-web/app/views/main/jobs_view.js

@@ -19,7 +19,7 @@
 var App = require('app');
 var filters = require('views/common/filter_view');
 
-App.MainJobsView = App.MainAppsView.extend({
+App.MainJobsView = Em.View.extend({
   templateName: require('templates/main/jobs'),
 
   showNumberOfJobs: Em.Select.extend({
@@ -39,6 +39,50 @@ App.MainJobsView = App.MainAppsView.extend({
     valueBinding: "controller.filterObject.id"
   }),
 
+  wrapSorting: Ember.View.extend({
+    tagName: 'tr'
+  }),
+
+  sortingColumns: Ember.View.extend({
+    tagName: 'th',
+    classNameBindings: ['class', 'widthClass'],
+    class: "sorting",
+    widthClass: "",
+    content: null,
+    defaultColumn: 8,
+
+    didInsertElement: function () {
+      this.set("widthClass", "col" + this.get('content.index'));
+      if (this.get('content.index') == this.get('defaultColumn')) {
+        this.setControllerObj(this.content.index, "DESC");
+        this.set("class", "sorting_desc");
+      }
+    },
+    click: function (event) {
+      console.log(this.get('class'));
+      if (this.get('class') == "sorting") {
+        this.resetSortClass();
+        this.setControllerObj(this.get('content.index'), "ASC");
+        this.set("class", "sorting_asc");
+      } else if (this.get('class') == "sorting_asc") {
+        this.setControllerObj(this.get('content.index'), "DESC");
+        this.set("class", "sorting_desc");
+      } else if (this.get('class') == "sorting_desc") {
+        this.setControllerObj(this.get('content.index'), "ASC");
+        this.set("class", "sorting_asc");
+      }
+    },
+    resetSortClass: function () {
+      this.get("parentView.childViews").map(function (a, e) {
+        a.get("childViews")[0].set("class", "sorting")
+      });
+    },
+    setControllerObj: function (col, dir) {
+      this.set("controller.filterObject.iSortCol_0", col);
+      this.set("controller.filterObject.sSortDir_0", dir);
+    }
+  }),
+
   /**
    * Filter-list for User.
    * Based on <code>filters</code> library

ambari-web/app/views/main/menu.js

@@ -39,7 +39,7 @@ App.MainMenuView = Em.CollectionView.extend({
 
     if (!App.get('isHadoop2Stack')) {
       result.push({ label:Em.I18n.t('menu.item.jobs'), routing:'apps'});
-    }else if (App.supports.jobs) {
+    } else if(App.supports.jobs) {
       result.push({ label:Em.I18n.t('menu.item.jobs'), routing:'jobs'});
     }