tezDag-name-to-id.json 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463
  1. {
  2. "entities": [
  3. {
  4. "starttime": 1393031617162,
  5. "events": [
  6. {
  7. "timestamp": 1393031624957,
  8. "eventtype": "DAG_FINISHED",
  9. "eventinfo": {}
  10. },
  11. {
  12. "timestamp": 1393031617405,
  13. "eventtype": "DAG_STARTED",
  14. "eventinfo": {}
  15. },
  16. {
  17. "timestamp": 1393031617401,
  18. "eventtype": "DAG_INITIALIZED",
  19. "eventinfo": {}
  20. },
  21. {
  22. "timestamp": 1393031617162,
  23. "eventtype": "DAG_SUBMITTED",
  24. "eventinfo": {}
  25. }
  26. ],
  27. "otherinfo": {
  28. "startTime": 1393031617405,
  29. "status": "SUCCEEDED",
  30. "initTime": 1393031617401,
  31. "timeTaken": 7552,
  32. "dagPlan": {
  33. "dagName": "root_20140221171313_8bdaf452-0a61-4cff-bd24-63662e2c3dcc:3",
  34. "edges": [
  35. {
  36. "edgeId": "167637864",
  37. "inputVertexName": "Map 2",
  38. "outputVertexName": "Reducer 3",
  39. "dataMovementType": "SCATTER_GATHER",
  40. "dataSourceType": "PERSISTED",
  41. "schedulingType": "SEQUENTIAL",
  42. "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileSortedOutput",
  43. "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy"
  44. },
  45. {
  46. "edgeId": "702054850",
  47. "inputVertexName": "Map 4",
  48. "outputVertexName": "Map 2",
  49. "dataMovementType": "BROADCAST",
  50. "dataSourceType": "PERSISTED",
  51. "schedulingType": "SEQUENTIAL",
  52. "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
  53. "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
  54. },
  55. {
  56. "edgeId": "1016858596",
  57. "inputVertexName": "Map 1",
  58. "outputVertexName": "Map 2",
  59. "dataMovementType": "BROADCAST",
  60. "dataSourceType": "PERSISTED",
  61. "schedulingType": "SEQUENTIAL",
  62. "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
  63. "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
  64. },
  65. {
  66. "edgeId": "1012906348",
  67. "inputVertexName": "Map 6",
  68. "outputVertexName": "Map 2",
  69. "dataMovementType": "BROADCAST",
  70. "dataSourceType": "PERSISTED",
  71. "schedulingType": "SEQUENTIAL",
  72. "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
  73. "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
  74. },
  75. {
  76. "edgeId": "591684997",
  77. "inputVertexName": "Map 5",
  78. "outputVertexName": "Map 2",
  79. "dataMovementType": "BROADCAST",
  80. "dataSourceType": "PERSISTED",
  81. "schedulingType": "SEQUENTIAL",
  82. "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
  83. "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
  84. }
  85. ],
  86. "vertices": [
  87. {
  88. "vertexName": "Reducer 3",
  89. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
  90. "inEdgeIds": [
  91. "167637864"
  92. ],
  93. "additionalOutputs": [
  94. {
  95. "name": "out_Reducer 3",
  96. "class": "org.apache.tez.mapreduce.output.MROutput"
  97. }
  98. ]
  99. },
  100. {
  101. "vertexName": "Map 2",
  102. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
  103. "inEdgeIds": [
  104. "1012906348",
  105. "591684997",
  106. "702054850",
  107. "1016858596"
  108. ],
  109. "outEdgeIds": [
  110. "167637864"
  111. ],
  112. "additionalInputs": [
  113. {
  114. "name": "store_sales",
  115. "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
  116. "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
  117. }
  118. ]
  119. },
  120. {
  121. "vertexName": "Map 6",
  122. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
  123. "outEdgeIds": [
  124. "1012906348"
  125. ],
  126. "additionalInputs": [
  127. {
  128. "name": "date_dim",
  129. "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
  130. "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
  131. }
  132. ]
  133. },
  134. {
  135. "vertexName": "Map 5",
  136. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
  137. "outEdgeIds": [
  138. "591684997"
  139. ],
  140. "additionalInputs": [
  141. {
  142. "name": "customer_demographics",
  143. "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
  144. "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
  145. }
  146. ]
  147. },
  148. {
  149. "vertexName": "Map 4",
  150. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
  151. "outEdgeIds": [
  152. "702054850"
  153. ],
  154. "additionalInputs": [
  155. {
  156. "name": "item",
  157. "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
  158. "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
  159. }
  160. ]
  161. },
  162. {
  163. "vertexName": "Map 1",
  164. "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
  165. "outEdgeIds": [
  166. "1016858596"
  167. ],
  168. "additionalInputs": [
  169. {
  170. "name": "store",
  171. "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
  172. "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
  173. }
  174. ]
  175. }
  176. ],
  177. "version": 1
  178. },
  179. "endTime": 1393031624957,
  180. "diagnostics": "",
  181. "counters": {
  182. "counterGroups": [
  183. {
  184. "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
  185. "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
  186. "counters": [
  187. {
  188. "counterName": "TOTAL_LAUNCHED_TASKS",
  189. "counterDisplayName": "TOTAL_LAUNCHED_TASKS",
  190. "counterValue": 180
  191. },
  192. {
  193. "counterName": "DATA_LOCAL_TASKS",
  194. "counterDisplayName": "DATA_LOCAL_TASKS",
  195. "counterValue": 162
  196. },
  197. {
  198. "counterName": "RACK_LOCAL_TASKS",
  199. "counterDisplayName": "RACK_LOCAL_TASKS",
  200. "counterValue": 17
  201. }
  202. ]
  203. },
  204. {
  205. "counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
  206. "counterGroupDisplayName": "File System Counters",
  207. "counters": [
  208. {
  209. "counterName": "FILE_BYTES_READ",
  210. "counterDisplayName": "FILE: BYTES_READ",
  211. "counterValue": 0
  212. },
  213. {
  214. "counterName": "FILE_BYTES_WRITTEN",
  215. "counterDisplayName": "FILE: BYTES_WRITTEN",
  216. "counterValue": 6380254
  217. },
  218. {
  219. "counterName": "FILE_READ_OPS",
  220. "counterDisplayName": "FILE: READ_OPS",
  221. "counterValue": 0
  222. },
  223. {
  224. "counterName": "FILE_LARGE_READ_OPS",
  225. "counterDisplayName": "FILE: LARGE_READ_OPS",
  226. "counterValue": 0
  227. },
  228. {
  229. "counterName": "FILE_WRITE_OPS",
  230. "counterDisplayName": "FILE: WRITE_OPS",
  231. "counterValue": 0
  232. },
  233. {
  234. "counterName": "HDFS_BYTES_READ",
  235. "counterDisplayName": "HDFS: BYTES_READ",
  236. "counterValue": 1291686552
  237. },
  238. {
  239. "counterName": "HDFS_BYTES_WRITTEN",
  240. "counterDisplayName": "HDFS: BYTES_WRITTEN",
  241. "counterValue": 8917
  242. },
  243. {
  244. "counterName": "HDFS_READ_OPS",
  245. "counterDisplayName": "HDFS: READ_OPS",
  246. "counterValue": 14144
  247. },
  248. {
  249. "counterName": "HDFS_LARGE_READ_OPS",
  250. "counterDisplayName": "HDFS: LARGE_READ_OPS",
  251. "counterValue": 0
  252. },
  253. {
  254. "counterName": "HDFS_WRITE_OPS",
  255. "counterDisplayName": "HDFS: WRITE_OPS",
  256. "counterValue": 2
  257. }
  258. ]
  259. },
  260. {
  261. "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
  262. "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
  263. "counters": [
  264. {
  265. "counterName": "MAP_INPUT_RECORDS",
  266. "counterDisplayName": "MAP_INPUT_RECORDS",
  267. "counterValue": 110379
  268. },
  269. {
  270. "counterName": "MAP_OUTPUT_RECORDS",
  271. "counterDisplayName": "MAP_OUTPUT_RECORDS",
  272. "counterValue": 394097
  273. },
  274. {
  275. "counterName": "MAP_OUTPUT_BYTES",
  276. "counterDisplayName": "MAP_OUTPUT_BYTES",
  277. "counterValue": 34555222
  278. },
  279. {
  280. "counterName": "MAP_OUTPUT_MATERIALIZED_BYTES",
  281. "counterDisplayName": "MAP_OUTPUT_MATERIALIZED_BYTES",
  282. "counterValue": 6136234
  283. },
  284. {
  285. "counterName": "COMBINE_INPUT_RECORDS",
  286. "counterDisplayName": "COMBINE_INPUT_RECORDS",
  287. "counterValue": 0
  288. },
  289. {
  290. "counterName": "REDUCE_INPUT_GROUPS",
  291. "counterDisplayName": "REDUCE_INPUT_GROUPS",
  292. "counterValue": 104
  293. },
  294. {
  295. "counterName": "REDUCE_SHUFFLE_BYTES",
  296. "counterDisplayName": "REDUCE_SHUFFLE_BYTES",
  297. "counterValue": 6136234
  298. },
  299. {
  300. "counterName": "REDUCE_INPUT_RECORDS",
  301. "counterDisplayName": "REDUCE_INPUT_RECORDS",
  302. "counterValue": 1747
  303. },
  304. {
  305. "counterName": "SPILLED_RECORDS",
  306. "counterDisplayName": "SPILLED_RECORDS",
  307. "counterValue": 394097
  308. },
  309. {
  310. "counterName": "SHUFFLED_MAPS",
  311. "counterDisplayName": "SHUFFLED_MAPS",
  312. "counterValue": 170
  313. },
  314. {
  315. "counterName": "FAILED_SHUFFLE",
  316. "counterDisplayName": "FAILED_SHUFFLE",
  317. "counterValue": 0
  318. },
  319. {
  320. "counterName": "MERGED_MAP_OUTPUTS",
  321. "counterDisplayName": "MERGED_MAP_OUTPUTS",
  322. "counterValue": 170
  323. },
  324. {
  325. "counterName": "GC_TIME_MILLIS",
  326. "counterDisplayName": "GC_TIME_MILLIS",
  327. "counterValue": 9277
  328. },
  329. {
  330. "counterName": "CPU_MILLISECONDS",
  331. "counterDisplayName": "CPU_MILLISECONDS",
  332. "counterValue": -2881340
  333. },
  334. {
  335. "counterName": "PHYSICAL_MEMORY_BYTES",
  336. "counterDisplayName": "PHYSICAL_MEMORY_BYTES",
  337. "counterValue": 238651179008
  338. },
  339. {
  340. "counterName": "VIRTUAL_MEMORY_BYTES",
  341. "counterDisplayName": "VIRTUAL_MEMORY_BYTES",
  342. "counterValue": 840369852416
  343. },
  344. {
  345. "counterName": "COMMITTED_HEAP_BYTES",
  346. "counterDisplayName": "COMMITTED_HEAP_BYTES",
  347. "counterValue": 273071931392
  348. }
  349. ]
  350. },
  351. {
  352. "counterGroupName": "HIVE",
  353. "counterGroupDisplayName": "HIVE",
  354. "counters": [
  355. {
  356. "counterName": "CREATED_FILES",
  357. "counterDisplayName": "CREATED_FILES",
  358. "counterValue": 1
  359. }
  360. ]
  361. },
  362. {
  363. "counterGroupName": "Shuffle Errors",
  364. "counterGroupDisplayName": "Shuffle Errors",
  365. "counters": [
  366. {
  367. "counterName": "BAD_ID",
  368. "counterDisplayName": "BAD_ID",
  369. "counterValue": 0
  370. },
  371. {
  372. "counterName": "CONNECTION",
  373. "counterDisplayName": "CONNECTION",
  374. "counterValue": 0
  375. },
  376. {
  377. "counterName": "IO_ERROR",
  378. "counterDisplayName": "IO_ERROR",
  379. "counterValue": 0
  380. },
  381. {
  382. "counterName": "WRONG_LENGTH",
  383. "counterDisplayName": "WRONG_LENGTH",
  384. "counterValue": 0
  385. },
  386. {
  387. "counterName": "WRONG_MAP",
  388. "counterDisplayName": "WRONG_MAP",
  389. "counterValue": 0
  390. },
  391. {
  392. "counterName": "WRONG_REDUCE",
  393. "counterDisplayName": "WRONG_REDUCE",
  394. "counterValue": 0
  395. }
  396. ]
  397. },
  398. {
  399. "counterGroupName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
  400. "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
  401. "counters": [
  402. {
  403. "counterName": "FILTERED",
  404. "counterDisplayName": "FILTERED",
  405. "counterValue": 0
  406. },
  407. {
  408. "counterName": "PASSED",
  409. "counterDisplayName": "PASSED",
  410. "counterValue": 0
  411. }
  412. ]
  413. },
  414. {
  415. "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
  416. "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
  417. "counters": [
  418. {
  419. "counterName": "DESERIALIZE_ERRORS",
  420. "counterDisplayName": "DESERIALIZE_ERRORS",
  421. "counterValue": 0
  422. }
  423. ]
  424. },
  425. {
  426. "counterGroupName": "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
  427. "counterGroupDisplayName": "File Input Format Counters ",
  428. "counters": [
  429. {
  430. "counterName": "BYTES_READ",
  431. "counterDisplayName": "Bytes Read",
  432. "counterValue": 0
  433. }
  434. ]
  435. },
  436. {
  437. "counterGroupName": "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
  438. "counterGroupDisplayName": "File Output Format Counters ",
  439. "counters": [
  440. {
  441. "counterName": "BYTES_WRITTEN",
  442. "counterDisplayName": "Bytes Written",
  443. "counterValue": 0
  444. }
  445. ]
  446. }
  447. ]
  448. }
  449. },
  450. "primaryfilters": {
  451. "dagName": [
  452. null
  453. ],
  454. "user": [
  455. null
  456. ]
  457. },
  458. "entity": "dag_1393012407618_0059_3",
  459. "entitytype": "TEZ_DAG_ID",
  460. "relatedentities": {}
  461. }
  462. ]
  463. }