123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463 |
- {
- "entities": [
- {
- "starttime": 1393031617162,
- "events": [
- {
- "timestamp": 1393031624957,
- "eventtype": "DAG_FINISHED",
- "eventinfo": {}
- },
- {
- "timestamp": 1393031617405,
- "eventtype": "DAG_STARTED",
- "eventinfo": {}
- },
- {
- "timestamp": 1393031617401,
- "eventtype": "DAG_INITIALIZED",
- "eventinfo": {}
- },
- {
- "timestamp": 1393031617162,
- "eventtype": "DAG_SUBMITTED",
- "eventinfo": {}
- }
- ],
- "otherinfo": {
- "startTime": 1393031617405,
- "status": "SUCCEEDED",
- "initTime": 1393031617401,
- "timeTaken": 7552,
- "dagPlan": {
- "dagName": "root_20140221171313_8bdaf452-0a61-4cff-bd24-63662e2c3dcc:3",
- "edges": [
- {
- "edgeId": "167637864",
- "inputVertexName": "Map 2",
- "outputVertexName": "Reducer 3",
- "dataMovementType": "SCATTER_GATHER",
- "dataSourceType": "PERSISTED",
- "schedulingType": "SEQUENTIAL",
- "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileSortedOutput",
- "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy"
- },
- {
- "edgeId": "702054850",
- "inputVertexName": "Map 4",
- "outputVertexName": "Map 2",
- "dataMovementType": "BROADCAST",
- "dataSourceType": "PERSISTED",
- "schedulingType": "SEQUENTIAL",
- "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
- "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
- },
- {
- "edgeId": "1016858596",
- "inputVertexName": "Map 1",
- "outputVertexName": "Map 2",
- "dataMovementType": "BROADCAST",
- "dataSourceType": "PERSISTED",
- "schedulingType": "SEQUENTIAL",
- "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
- "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
- },
- {
- "edgeId": "1012906348",
- "inputVertexName": "Map 6",
- "outputVertexName": "Map 2",
- "dataMovementType": "BROADCAST",
- "dataSourceType": "PERSISTED",
- "schedulingType": "SEQUENTIAL",
- "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
- "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
- },
- {
- "edgeId": "591684997",
- "inputVertexName": "Map 5",
- "outputVertexName": "Map 2",
- "dataMovementType": "BROADCAST",
- "dataSourceType": "PERSISTED",
- "schedulingType": "SEQUENTIAL",
- "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
- "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
- }
- ],
- "vertices": [
- {
- "vertexName": "Reducer 3",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
- "inEdgeIds": [
- "167637864"
- ],
- "additionalOutputs": [
- {
- "name": "out_Reducer 3",
- "class": "org.apache.tez.mapreduce.output.MROutput"
- }
- ]
- },
- {
- "vertexName": "Map 2",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
- "inEdgeIds": [
- "1012906348",
- "591684997",
- "702054850",
- "1016858596"
- ],
- "outEdgeIds": [
- "167637864"
- ],
- "additionalInputs": [
- {
- "name": "store_sales",
- "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
- "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
- }
- ]
- },
- {
- "vertexName": "Map 6",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
- "outEdgeIds": [
- "1012906348"
- ],
- "additionalInputs": [
- {
- "name": "date_dim",
- "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
- "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
- }
- ]
- },
- {
- "vertexName": "Map 5",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
- "outEdgeIds": [
- "591684997"
- ],
- "additionalInputs": [
- {
- "name": "customer_demographics",
- "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
- "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
- }
- ]
- },
- {
- "vertexName": "Map 4",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
- "outEdgeIds": [
- "702054850"
- ],
- "additionalInputs": [
- {
- "name": "item",
- "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
- "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
- }
- ]
- },
- {
- "vertexName": "Map 1",
- "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
- "outEdgeIds": [
- "1016858596"
- ],
- "additionalInputs": [
- {
- "name": "store",
- "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
- "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
- }
- ]
- }
- ],
- "version": 1
- },
- "endTime": 1393031624957,
- "diagnostics": "",
- "counters": {
- "counterGroups": [
- {
- "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
- "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
- "counters": [
- {
- "counterName": "TOTAL_LAUNCHED_TASKS",
- "counterDisplayName": "TOTAL_LAUNCHED_TASKS",
- "counterValue": 180
- },
- {
- "counterName": "DATA_LOCAL_TASKS",
- "counterDisplayName": "DATA_LOCAL_TASKS",
- "counterValue": 162
- },
- {
- "counterName": "RACK_LOCAL_TASKS",
- "counterDisplayName": "RACK_LOCAL_TASKS",
- "counterValue": 17
- }
- ]
- },
- {
- "counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
- "counterGroupDisplayName": "File System Counters",
- "counters": [
- {
- "counterName": "FILE_BYTES_READ",
- "counterDisplayName": "FILE: BYTES_READ",
- "counterValue": 0
- },
- {
- "counterName": "FILE_BYTES_WRITTEN",
- "counterDisplayName": "FILE: BYTES_WRITTEN",
- "counterValue": 6380254
- },
- {
- "counterName": "FILE_READ_OPS",
- "counterDisplayName": "FILE: READ_OPS",
- "counterValue": 0
- },
- {
- "counterName": "FILE_LARGE_READ_OPS",
- "counterDisplayName": "FILE: LARGE_READ_OPS",
- "counterValue": 0
- },
- {
- "counterName": "FILE_WRITE_OPS",
- "counterDisplayName": "FILE: WRITE_OPS",
- "counterValue": 0
- },
- {
- "counterName": "HDFS_BYTES_READ",
- "counterDisplayName": "HDFS: BYTES_READ",
- "counterValue": 1291686552
- },
- {
- "counterName": "HDFS_BYTES_WRITTEN",
- "counterDisplayName": "HDFS: BYTES_WRITTEN",
- "counterValue": 8917
- },
- {
- "counterName": "HDFS_READ_OPS",
- "counterDisplayName": "HDFS: READ_OPS",
- "counterValue": 14144
- },
- {
- "counterName": "HDFS_LARGE_READ_OPS",
- "counterDisplayName": "HDFS: LARGE_READ_OPS",
- "counterValue": 0
- },
- {
- "counterName": "HDFS_WRITE_OPS",
- "counterDisplayName": "HDFS: WRITE_OPS",
- "counterValue": 2
- }
- ]
- },
- {
- "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
- "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
- "counters": [
- {
- "counterName": "MAP_INPUT_RECORDS",
- "counterDisplayName": "MAP_INPUT_RECORDS",
- "counterValue": 110379
- },
- {
- "counterName": "MAP_OUTPUT_RECORDS",
- "counterDisplayName": "MAP_OUTPUT_RECORDS",
- "counterValue": 394097
- },
- {
- "counterName": "MAP_OUTPUT_BYTES",
- "counterDisplayName": "MAP_OUTPUT_BYTES",
- "counterValue": 34555222
- },
- {
- "counterName": "MAP_OUTPUT_MATERIALIZED_BYTES",
- "counterDisplayName": "MAP_OUTPUT_MATERIALIZED_BYTES",
- "counterValue": 6136234
- },
- {
- "counterName": "COMBINE_INPUT_RECORDS",
- "counterDisplayName": "COMBINE_INPUT_RECORDS",
- "counterValue": 0
- },
- {
- "counterName": "REDUCE_INPUT_GROUPS",
- "counterDisplayName": "REDUCE_INPUT_GROUPS",
- "counterValue": 104
- },
- {
- "counterName": "REDUCE_SHUFFLE_BYTES",
- "counterDisplayName": "REDUCE_SHUFFLE_BYTES",
- "counterValue": 6136234
- },
- {
- "counterName": "REDUCE_INPUT_RECORDS",
- "counterDisplayName": "REDUCE_INPUT_RECORDS",
- "counterValue": 1747
- },
- {
- "counterName": "SPILLED_RECORDS",
- "counterDisplayName": "SPILLED_RECORDS",
- "counterValue": 394097
- },
- {
- "counterName": "SHUFFLED_MAPS",
- "counterDisplayName": "SHUFFLED_MAPS",
- "counterValue": 170
- },
- {
- "counterName": "FAILED_SHUFFLE",
- "counterDisplayName": "FAILED_SHUFFLE",
- "counterValue": 0
- },
- {
- "counterName": "MERGED_MAP_OUTPUTS",
- "counterDisplayName": "MERGED_MAP_OUTPUTS",
- "counterValue": 170
- },
- {
- "counterName": "GC_TIME_MILLIS",
- "counterDisplayName": "GC_TIME_MILLIS",
- "counterValue": 9277
- },
- {
- "counterName": "CPU_MILLISECONDS",
- "counterDisplayName": "CPU_MILLISECONDS",
- "counterValue": -2881340
- },
- {
- "counterName": "PHYSICAL_MEMORY_BYTES",
- "counterDisplayName": "PHYSICAL_MEMORY_BYTES",
- "counterValue": 238651179008
- },
- {
- "counterName": "VIRTUAL_MEMORY_BYTES",
- "counterDisplayName": "VIRTUAL_MEMORY_BYTES",
- "counterValue": 840369852416
- },
- {
- "counterName": "COMMITTED_HEAP_BYTES",
- "counterDisplayName": "COMMITTED_HEAP_BYTES",
- "counterValue": 273071931392
- }
- ]
- },
- {
- "counterGroupName": "HIVE",
- "counterGroupDisplayName": "HIVE",
- "counters": [
- {
- "counterName": "CREATED_FILES",
- "counterDisplayName": "CREATED_FILES",
- "counterValue": 1
- }
- ]
- },
- {
- "counterGroupName": "Shuffle Errors",
- "counterGroupDisplayName": "Shuffle Errors",
- "counters": [
- {
- "counterName": "BAD_ID",
- "counterDisplayName": "BAD_ID",
- "counterValue": 0
- },
- {
- "counterName": "CONNECTION",
- "counterDisplayName": "CONNECTION",
- "counterValue": 0
- },
- {
- "counterName": "IO_ERROR",
- "counterDisplayName": "IO_ERROR",
- "counterValue": 0
- },
- {
- "counterName": "WRONG_LENGTH",
- "counterDisplayName": "WRONG_LENGTH",
- "counterValue": 0
- },
- {
- "counterName": "WRONG_MAP",
- "counterDisplayName": "WRONG_MAP",
- "counterValue": 0
- },
- {
- "counterName": "WRONG_REDUCE",
- "counterDisplayName": "WRONG_REDUCE",
- "counterValue": 0
- }
- ]
- },
- {
- "counterGroupName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
- "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
- "counters": [
- {
- "counterName": "FILTERED",
- "counterDisplayName": "FILTERED",
- "counterValue": 0
- },
- {
- "counterName": "PASSED",
- "counterDisplayName": "PASSED",
- "counterValue": 0
- }
- ]
- },
- {
- "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
- "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
- "counters": [
- {
- "counterName": "DESERIALIZE_ERRORS",
- "counterDisplayName": "DESERIALIZE_ERRORS",
- "counterValue": 0
- }
- ]
- },
- {
- "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
- "counterGroupDisplayName": "File Input Format Counters ",
- "counters": [
- {
- "counterName": "BYTES_READ",
- "counterDisplayName": "Bytes Read",
- "counterValue": 0
- }
- ]
- },
- {
- "counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counterGroupDisplayName": "File Output Format Counters ",
- "counters": [
- {
- "counterName": "BYTES_WRITTEN",
- "counterDisplayName": "Bytes Written",
- "counterValue": 0
- }
- ]
- }
- ]
- }
- },
- "primaryfilters": {
- "dagName": [
- null
- ],
- "user": [
- null
- ]
- },
- "entity": "dag_1393012407618_0059_3",
- "entitytype": "TEZ_DAG_ID",
- "relatedentities": {}
- }
- ]
- }
|