HistoryServerRest.apt.vm 76 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651
  1. ~~ Licensed under the Apache License, Version 2.0 (the "License");
  2. ~~ you may not use this file except in compliance with the License.
  3. ~~ You may obtain a copy of the License at
  4. ~~
  5. ~~ http://www.apache.org/licenses/LICENSE-2.0
  6. ~~
  7. ~~ Unless required by applicable law or agreed to in writing, software
  8. ~~ distributed under the License is distributed on an "AS IS" BASIS,
  9. ~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. ~~ See the License for the specific language governing permissions and
  11. ~~ limitations under the License. See accompanying LICENSE file.
  12. ---
  13. History Server REST API's.
  14. ---
  15. ---
  16. ${maven.build.timestamp}
  17. History Server REST API's.
  18. \[ {{{./index.html}Go Back}} \]
  19. %{toc|section=1|fromDepth=0|toDepth=3}
  20. * Overview
  21. The history server REST API's allow the user to get status on finished applications. Currently it only supports MapReduce and provides information on finished jobs.
  22. * History Server Information API
  23. The history server information resource provides overall information about the history server.
  24. ** URI
  25. Both of the following URI's give you the history server information, from an application id identified by the appid value.
  26. ------
  27. * http://<history server http address:port>/ws/v1/history
  28. * http://<history server http address:port>/ws/v1/history/info
  29. ------
  30. ** HTTP Operations Supported
  31. ------
  32. * GET
  33. ------
  34. ** Query Parameters Supported
  35. ------
  36. None
  37. ------
  38. ** Elements of the <historyInfo> object
  39. *---------------+--------------+-------------------------------+
  40. || Item || Data Type || Description |
  41. *---------------+--------------+-------------------------------+
  42. | hadoopVersion | string | Version of hadoop common |
  43. *---------------+--------------+-------------------------------+
  44. | hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum |
  45. *---------------+--------------+-------------------------------+
  46. | hadoopVersionBuiltOn | string | Timestamp when hadoop common was built |
  47. *---------------+--------------+-------------------------------+
  48. ** Response Examples
  49. <<JSON response>>
  50. HTTP Request:
  51. ------
  52. GET http://<history server http address:port>/ws/v1/history/info
  53. ------
  54. Response Header:
  55. +---+
  56. HTTP/1.1 200 OK
  57. Content-Type: application/json
  58. Transfer-Encoding: chunked
  59. Server: Jetty(6.1.26)
  60. +---+
  61. Response Body:
  62. +---+
  63. {
  64. "historyInfo" : {
  65. "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012",
  66. "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7",
  67. "hadoopVersion" : "0.23.1-SNAPSHOT"
  68. }
  69. }
  70. +---+
  71. <<XML response>>
  72. HTTP Request:
  73. -----
  74. GET http://<history server http address:port>/ws/v1/history/info
  75. Accept: application/xml
  76. -----
  77. Response Header:
  78. +---+
  79. HTTP/1.1 200 OK
  80. Content-Type: application/xml
  81. Content-Length: 330
  82. Server: Jetty(6.1.26)
  83. +---+
  84. Response Body:
  85. +---+
  86. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  87. <historyInfo>
  88. <hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion>
  89. <hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion>
  90. <hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn>
  91. </historyInfo>
  92. +---+
  93. * MapReduce API's
  94. The following list of resources apply to MapReduce.
  95. ** Jobs API
  96. The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters
  97. *** URI
  98. ------
  99. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs
  100. ------
  101. *** HTTP Operations Supported
  102. ------
  103. * GET
  104. ------
  105. *** Query Parameters Supported
  106. Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all jobs that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity.
  107. ------
  108. * user - user name
  109. * queue - queue name
  110. * limit - total number of app objects to be returned
  111. * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch
  112. * startedTimeEnd - jobs with start time ending with this time, specified in ms since epoch
  113. * finishedTimeBegin - jobs with finish time beginning with this time, specified in ms since epoch
  114. * finishedTimeEnd - jobs with finish time ending with this time, specified in ms since epoch
  115. ------
  116. *** Elements of the <jobs> object
  117. When you make a request for the list of jobs, the information will be returned as an array of job objects.
  118. See also {{Job API}} for syntax of the job object. Except this is a subset of a full job. Only startTime,
  119. finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are
  120. returned.
  121. *---------------+--------------+-------------------------------+
  122. || Item || Data Type || Description |
  123. *---------------+--------------+-------------------------------+
  124. | job | array of job objects(json)/zero or more job objects(XML) | The collection of job objects |
  125. *---------------+--------------+-------------------------------+
  126. *** Response Examples
  127. <<JSON response>>
  128. HTTP Request:
  129. ------
  130. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
  131. ------
  132. Response Header:
  133. +---+
  134. HTTP/1.1 200 OK
  135. Content-Type: application/json
  136. Transfer-Encoding: chunked
  137. Server: Jetty(6.1.26)
  138. +---+
  139. Response Body:
  140. +---+
  141. {
  142. "jobs" : {
  143. "job" : [
  144. {
  145. "state" : "SUCCEEDED",
  146. "user" : "user1",
  147. "reducesTotal" : 1,
  148. "mapsCompleted" : 1,
  149. "startTime" : 1326381344489,
  150. "id" : "job_1326381300833_1_1",
  151. "name" : "word count",
  152. "reducesCompleted" : 1,
  153. "mapsTotal" : 1,
  154. "queue" : "default",
  155. "finishTime" : 1326381356010
  156. },
  157. {
  158. "state" : "SUCCEEDED",
  159. "user" : "user1",
  160. "reducesTotal" : 1,
  161. "mapsCompleted" : 1,
  162. "startTime" : 1326381446529,
  163. "id" : "job_1326381300833_2_2",
  164. "name" : "Sleep job",
  165. "reducesCompleted" : 1,
  166. "mapsTotal" : 1,
  167. "queue" : "default",
  168. "finishTime" : 1326381582106
  169. }
  170. ]
  171. }
  172. }
  173. +---+
  174. <<XML response>>
  175. HTTP Request:
  176. ------
  177. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
  178. Accept: application/xml
  179. ------
  180. Response Header:
  181. +---+
  182. HTTP/1.1 200 OK
  183. Content-Type: application/xml
  184. Content-Length: 1922
  185. Server: Jetty(6.1.26)
  186. +---+
  187. Response Body:
  188. +---+
  189. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  190. <jobs>
  191. <job>
  192. <startTime>1326381344489</startTime>
  193. <finishTime>1326381356010</finishTime>
  194. <id>job_1326381300833_1_1</id>
  195. <name>word count</name>
  196. <queue>default</queue>
  197. <user>user1</user>
  198. <state>SUCCEEDED</state>
  199. <mapsTotal>1</mapsTotal>
  200. <mapsCompleted>1</mapsCompleted>
  201. <reducesTotal>1</reducesTotal>
  202. <reducesCompleted>1</reducesCompleted>
  203. </job>
  204. <job>
  205. <startTime>1326381446529</startTime>
  206. <finishTime>1326381582106</finishTime>
  207. <id>job_1326381300833_2_2</id>
  208. <name>Sleep job</name>
  209. <queue>default</queue>
  210. <user>user1</user>
  211. <state>SUCCEEDED</state>
  212. <mapsTotal>1</mapsTotal>
  213. <mapsCompleted>1</mapsCompleted>
  214. <reducesTotal>1</reducesTotal>
  215. <reducesCompleted>1</reducesCompleted>
  216. </job>
  217. </jobs>
  218. +---+
  219. ** {Job API}
  220. A Job resource contains information about a particular job identified by {jobid}.
  221. *** URI
  222. ------
  223. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}
  224. ------
  225. *** HTTP Operations Supported
  226. ------
  227. * GET
  228. ------
  229. *** Query Parameters Supported
  230. ------
  231. None
  232. ------
  233. *** Elements of the <job> object
  234. *---------------+--------------+-------------------------------+
  235. || Item || Data Type || Description |
  236. *---------------+--------------+-------------------------------+
  237. | id | string | The job id|
  238. *---------------+--------------+-------------------------------+
  239. | name | string | The job name |
  240. *---------------+--------------+-------------------------------+
  241. | queue | string | The queue the job was submitted to|
  242. *---------------+--------------+-------------------------------+
  243. | user | string | The user name |
  244. *---------------+--------------+-------------------------------+
  245. | state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR|
  246. *---------------+--------------+-------------------------------+
  247. | diagnostics | string | A diagnostic message |
  248. *---------------+--------------+-------------------------------+
  249. | startTime | long | The time the job started (in ms since epoch)|
  250. *---------------+--------------+-------------------------------+
  251. | finishTime | long | The time the job finished (in ms since epoch)|
  252. *---------------+--------------+-------------------------------+
  253. | mapsTotal | int | The total number of maps |
  254. *---------------+--------------+-------------------------------+
  255. | mapsCompleted | int | The number of completed maps |
  256. *---------------+--------------+-------------------------------+
  257. | reducesTotal | int | The total number of reduces |
  258. *---------------+--------------+-------------------------------+
  259. | reducesCompleted | int | The number of completed reduces|
  260. *---------------+--------------+-------------------------------+
  261. | uberized | boolean | Indicates if the job was an uber job - ran completely in the application master|
  262. *---------------+--------------+-------------------------------+
  263. | avgMapTime | long | The average time of a map task (in ms)|
  264. *---------------+--------------+-------------------------------+
  265. | avgReduceTime | long | The average time of the reduce (in ms)|
  266. *---------------+--------------+-------------------------------+
  267. | avgShuffleTime | long | The average time of the shuffle (in ms)|
  268. *---------------+--------------+-------------------------------+
  269. | avgMergeTime | long | The average time of the merge (in ms)|
  270. *---------------+--------------+-------------------------------+
  271. | failedReduceAttempts | int | The number of failed reduce attempts |
  272. *---------------+--------------+-------------------------------+
  273. | killedReduceAttempts | int | The number of killed reduce attempts |
  274. *---------------+--------------+-------------------------------+
  275. | successfulReduceAttempts | int | The number of successful reduce attempts |
  276. *---------------+--------------+-------------------------------+
  277. | failedMapAttempts | int | The number of failed map attempts |
  278. *---------------+--------------+-------------------------------+
  279. | killedMapAttempts | int | The number of killed map attempts |
  280. *---------------+--------------+-------------------------------+
  281. | successfulMapAttempts | int | The number of successful map attempts |
  282. *---------------+--------------+-------------------------------+
  283. | acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects |
  284. *---------------+--------------+-------------------------------+
  285. ** Elements of the <acls> object
  286. *---------------+--------------+-------------------------------+
  287. || Item || Data Type || Description |
  288. *---------------+--------------+-------------------------------+
  289. | value | string | The acl value|
  290. *---------------+--------------+-------------------------------+
  291. | name | string | The acl name |
  292. *---------------+--------------+-------------------------------+
  293. *** Response Examples
  294. <<JSON response>>
  295. HTTP Request:
  296. ------
  297. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
  298. ------
  299. Response Header:
  300. +---+
  301. HTTP/1.1 200 OK
  302. Content-Type: application/json
  303. Server: Jetty(6.1.26)
  304. Content-Length: 720
  305. +---+
  306. Response Body:
  307. +---+
  308. {
  309. "job" : {
  310. "avgReduceTime" : 124961,
  311. "failedReduceAttempts" : 0,
  312. "state" : "SUCCEEDED",
  313. "successfulReduceAttempts" : 1,
  314. "acls" : [
  315. {
  316. "value" : " ",
  317. "name" : "mapreduce.job.acl-modify-job"
  318. },
  319. {
  320. "value" : " ",
  321. "name" : "mapreduce.job.acl-view-job"
  322. }
  323. ],
  324. "user" : "user1",
  325. "reducesTotal" : 1,
  326. "mapsCompleted" : 1,
  327. "startTime" : 1326381446529,
  328. "id" : "job_1326381300833_2_2",
  329. "avgMapTime" : 2638,
  330. "successfulMapAttempts" : 1,
  331. "name" : "Sleep job",
  332. "avgShuffleTime" : 2540,
  333. "reducesCompleted" : 1,
  334. "diagnostics" : "",
  335. "failedMapAttempts" : 0,
  336. "avgMergeTime" : 2589,
  337. "killedReduceAttempts" : 0,
  338. "mapsTotal" : 1,
  339. "queue" : "default",
  340. "uberized" : false,
  341. "killedMapAttempts" : 0,
  342. "finishTime" : 1326381582106
  343. }
  344. }
  345. +---+
  346. <<XML response>>
  347. HTTP Request:
  348. ------
  349. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
  350. Accept: application/xml
  351. ------
  352. Response Header:
  353. +---+
  354. HTTP/1.1 200 OK
  355. Content-Type: application/xml
  356. Content-Length: 983
  357. Server: Jetty(6.1.26)
  358. +---+
  359. Response Body:
  360. +---+
  361. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  362. <job>
  363. <startTime>1326381446529</startTime>
  364. <finishTime>1326381582106</finishTime>
  365. <id>job_1326381300833_2_2</id>
  366. <name>Sleep job</name>
  367. <queue>default</queue>
  368. <user>user1</user>
  369. <state>SUCCEEDED</state>
  370. <mapsTotal>1</mapsTotal>
  371. <mapsCompleted>1</mapsCompleted>
  372. <reducesTotal>1</reducesTotal>
  373. <reducesCompleted>1</reducesCompleted>
  374. <uberized>false</uberized>
  375. <diagnostics/>
  376. <avgMapTime>2638</avgMapTime>
  377. <avgReduceTime>124961</avgReduceTime>
  378. <avgShuffleTime>2540</avgShuffleTime>
  379. <avgMergeTime>2589</avgMergeTime>
  380. <failedReduceAttempts>0</failedReduceAttempts>
  381. <killedReduceAttempts>0</killedReduceAttempts>
  382. <successfulReduceAttempts>1</successfulReduceAttempts>
  383. <failedMapAttempts>0</failedMapAttempts>
  384. <killedMapAttempts>0</killedMapAttempts>
  385. <successfulMapAttempts>1</successfulMapAttempts>
  386. <acls>
  387. <name>mapreduce.job.acl-modify-job</name>
  388. <value> </value>
  389. </acls>
  390. <acls>
  391. <name>mapreduce.job.acl-view-job</name>
  392. <value> </value>
  393. </acls>
  394. </job>
  395. +---+
  396. ** Job Attempts API
  397. With the job attempts API, you can obtain a collection of resources that represent a job attempt. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects.
  398. *** URI
  399. ------
  400. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/jobattempts
  401. ------
  402. *** HTTP Operations Supported
  403. ------
  404. * GET
  405. ------
  406. *** Query Parameters Supported
  407. ------
  408. None
  409. ------
  410. *** Elements of the <jobAttempts> object
  411. When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects.
  412. jobAttempts:
  413. *---------------+--------------+-------------------------------+
  414. || Item || Data Type || Description |
  415. *---------------+--------------+-------------------------------+
  416. | jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects |
  417. *---------------+--------------+--------------------------------+
  418. *** Elements of the <jobAttempt> object
  419. *---------------+--------------+-------------------------------+
  420. || Item || Data Type || Description |
  421. *---------------+--------------+-------------------------------+
  422. | id | string | The job attempt id |
  423. *---------------+--------------+--------------------------------+
  424. | nodeId | string | The node id of the node the attempt ran on|
  425. *---------------+--------------+--------------------------------+
  426. | nodeHttpAddress | string | The node http address of the node the attempt ran on|
  427. *---------------+--------------+--------------------------------+
  428. | logsLink | string | The http link to the job attempt logs |
  429. *---------------+--------------+--------------------------------+
  430. | containerId | string | The id of the container for the job attempt |
  431. *---------------+--------------+--------------------------------+
  432. | startTime | long | The start time of the attempt (in ms since epoch)|
  433. *---------------+--------------+--------------------------------+
  434. *** Response Examples
  435. <<JSON response>>
  436. HTTP Request:
  437. ------
  438. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattempts
  439. ------
  440. Response Header:
  441. +---+
  442. HTTP/1.1 200 OK
  443. Content-Type: application/json
  444. Transfer-Encoding: chunked
  445. Server: Jetty(6.1.26)
  446. +---+
  447. Response Body:
  448. +---+
  449. {
  450. "jobAttempts" : {
  451. "jobAttempt" : [
  452. {
  453. "nodeId" : "host.domain.com:8041",
  454. "nodeHttpAddress" : "host.domain.com:8042",
  455. "startTime" : 1326381444693,
  456. "id" : 1,
  457. "logsLink" : "http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1",
  458. "containerId" : "container_1326381300833_0002_01_000001"
  459. }
  460. ]
  461. }
  462. }
  463. +---+
  464. <<XML response>>
  465. HTTP Request:
  466. ------
  467. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattmpts
  468. Accept: application/xml
  469. ------
  470. Response Header:
  471. +---+
  472. HTTP/1.1 200 OK
  473. Content-Type: application/xml
  474. Content-Length: 575
  475. Server: Jetty(6.1.26)
  476. +---+
  477. Response Body:
  478. +---+
  479. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  480. <jobAttempts>
  481. <jobAttempt>
  482. <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
  483. <nodeId>host.domain.com:8041</nodeId>
  484. <id>1</id>
  485. <startTime>1326381444693</startTime>
  486. <containerId>container_1326381300833_0002_01_000001</containerId>
  487. <logsLink>http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1</logsLink>
  488. </jobAttempt>
  489. </jobAttempts>
  490. +---+
  491. ** Job Counters API
  492. With the job counters API, you can object a collection of resources that represent al the counters for that job.
  493. *** URI
  494. ------
  495. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/counters
  496. ------
  497. *** HTTP Operations Supported
  498. ------
  499. * GET
  500. ------
  501. *** Query Parameters Supported
  502. ------
  503. None
  504. ------
  505. *** Elements of the <jobCounters> object
  506. *---------------+--------------+-------------------------------+
  507. || Item || Data Type || Description |
  508. *---------------+--------------+-------------------------------+
  509. | id | string | The job id |
  510. *---------------+--------------+-------------------------------+
  511. | counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
  512. *---------------+--------------+-------------------------------+
  513. *** Elements of the <counterGroup> objecs
  514. *---------------+--------------+-------------------------------+
  515. || Item || Data Type || Description |
  516. *---------------+--------------+-------------------------------+
  517. | counterGroupName | string | The name of the counter group |
  518. *---------------+--------------+-------------------------------+
  519. | counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
  520. *---------------+--------------+-------------------------------+
  521. *** Elements of the <counter> object
  522. *---------------+--------------+-------------------------------+
  523. || Item || Data Type || Description |
  524. *---------------+--------------+-------------------------------+
  525. | name | string | The name of the counter |
  526. *---------------+--------------+-------------------------------+
  527. | reduceCounterValue | long | The counter value of reduce tasks |
  528. *---------------+--------------+-------------------------------+
  529. | mapCounterValue | long | The counter value of map tasks |
  530. *---------------+--------------+-------------------------------+
  531. | totalCounterValue | long | The counter value of all tasks |
  532. *---------------+--------------+-------------------------------+
  533. *** Response Examples
  534. <<JSON response>>
  535. HTTP Request:
  536. ------
  537. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
  538. ------
  539. Response Header:
  540. +---+
  541. HTTP/1.1 200 OK
  542. Content-Type: application/json
  543. Transfer-Encoding: chunked
  544. Server: Jetty(6.1.26)
  545. +---+
  546. Response Body:
  547. +---+
  548. {
  549. "jobCounters" : {
  550. "id" : "job_1326381300833_2_2",
  551. "counterGroup" : [
  552. {
  553. "counterGroupName" : "Shuffle Errors",
  554. "counter" : [
  555. {
  556. "reduceCounterValue" : 0,
  557. "mapCounterValue" : 0,
  558. "totalCounterValue" : 0,
  559. "name" : "BAD_ID"
  560. },
  561. {
  562. "reduceCounterValue" : 0,
  563. "mapCounterValue" : 0,
  564. "totalCounterValue" : 0,
  565. "name" : "CONNECTION"
  566. },
  567. {
  568. "reduceCounterValue" : 0,
  569. "mapCounterValue" : 0,
  570. "totalCounterValue" : 0,
  571. "name" : "IO_ERROR"
  572. },
  573. {
  574. "reduceCounterValue" : 0,
  575. "mapCounterValue" : 0,
  576. "totalCounterValue" : 0,
  577. "name" : "WRONG_LENGTH"
  578. },
  579. {
  580. "reduceCounterValue" : 0,
  581. "mapCounterValue" : 0,
  582. "totalCounterValue" : 0,
  583. "name" : "WRONG_MAP"
  584. },
  585. {
  586. "reduceCounterValue" : 0,
  587. "mapCounterValue" : 0,
  588. "totalCounterValue" : 0,
  589. "name" : "WRONG_REDUCE"
  590. }
  591. ]
  592. },
  593. {
  594. "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
  595. "counter" : [
  596. {
  597. "reduceCounterValue" : 0,
  598. "mapCounterValue" : 0,
  599. "totalCounterValue" : 2483,
  600. "name" : "FILE_BYTES_READ"
  601. },
  602. {
  603. "reduceCounterValue" : 0,
  604. "mapCounterValue" : 0,
  605. "totalCounterValue" : 108525,
  606. "name" : "FILE_BYTES_WRITTEN"
  607. },
  608. {
  609. "reduceCounterValue" : 0,
  610. "mapCounterValue" : 0,
  611. "totalCounterValue" : 0,
  612. "name" : "FILE_READ_OPS"
  613. },
  614. {
  615. "reduceCounterValue" : 0,
  616. "mapCounterValue" : 0,
  617. "totalCounterValue" : 0,
  618. "name" : "FILE_LARGE_READ_OPS"
  619. },
  620. {
  621. "reduceCounterValue" : 0,
  622. "mapCounterValue" : 0,
  623. "totalCounterValue" : 0,
  624. "name" : "FILE_WRITE_OPS"
  625. },
  626. {
  627. "reduceCounterValue" : 0,
  628. "mapCounterValue" : 0,
  629. "totalCounterValue" : 48,
  630. "name" : "HDFS_BYTES_READ"
  631. },
  632. {
  633. "reduceCounterValue" : 0,
  634. "mapCounterValue" : 0,
  635. "totalCounterValue" : 0,
  636. "name" : "HDFS_BYTES_WRITTEN"
  637. },
  638. {
  639. "reduceCounterValue" : 0,
  640. "mapCounterValue" : 0,
  641. "totalCounterValue" : 1,
  642. "name" : "HDFS_READ_OPS"
  643. },
  644. {
  645. "reduceCounterValue" : 0,
  646. "mapCounterValue" : 0,
  647. "totalCounterValue" : 0,
  648. "name" : "HDFS_LARGE_READ_OPS"
  649. },
  650. {
  651. "reduceCounterValue" : 0,
  652. "mapCounterValue" : 0,
  653. "totalCounterValue" : 0,
  654. "name" : "HDFS_WRITE_OPS"
  655. }
  656. ]
  657. },
  658. {
  659. "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
  660. "counter" : [
  661. {
  662. "reduceCounterValue" : 0,
  663. "mapCounterValue" : 0,
  664. "totalCounterValue" : 1,
  665. "name" : "MAP_INPUT_RECORDS"
  666. },
  667. {
  668. "reduceCounterValue" : 0,
  669. "mapCounterValue" : 0,
  670. "totalCounterValue" : 1200,
  671. "name" : "MAP_OUTPUT_RECORDS"
  672. },
  673. {
  674. "reduceCounterValue" : 0,
  675. "mapCounterValue" : 0,
  676. "totalCounterValue" : 4800,
  677. "name" : "MAP_OUTPUT_BYTES"
  678. },
  679. {
  680. "reduceCounterValue" : 0,
  681. "mapCounterValue" : 0,
  682. "totalCounterValue" : 2235,
  683. "name" : "MAP_OUTPUT_MATERIALIZED_BYTES"
  684. },
  685. {
  686. "reduceCounterValue" : 0,
  687. "mapCounterValue" : 0,
  688. "totalCounterValue" : 48,
  689. "name" : "SPLIT_RAW_BYTES"
  690. },
  691. {
  692. "reduceCounterValue" : 0,
  693. "mapCounterValue" : 0,
  694. "totalCounterValue" : 0,
  695. "name" : "COMBINE_INPUT_RECORDS"
  696. },
  697. {
  698. "reduceCounterValue" : 0,
  699. "mapCounterValue" : 0,
  700. "totalCounterValue" : 0,
  701. "name" : "COMBINE_OUTPUT_RECORDS"
  702. },
  703. {
  704. "reduceCounterValue" : 0,
  705. "mapCounterValue" : 0,
  706. "totalCounterValue" : 1200,
  707. "name" : "REDUCE_INPUT_GROUPS"
  708. },
  709. {
  710. "reduceCounterValue" : 0,
  711. "mapCounterValue" : 0,
  712. "totalCounterValue" : 2235,
  713. "name" : "REDUCE_SHUFFLE_BYTES"
  714. },
  715. {
  716. "reduceCounterValue" : 0,
  717. "mapCounterValue" : 0,
  718. "totalCounterValue" : 1200,
  719. "name" : "REDUCE_INPUT_RECORDS"
  720. },
  721. {
  722. "reduceCounterValue" : 0,
  723. "mapCounterValue" : 0,
  724. "totalCounterValue" : 0,
  725. "name" : "REDUCE_OUTPUT_RECORDS"
  726. },
  727. {
  728. "reduceCounterValue" : 0,
  729. "mapCounterValue" : 0,
  730. "totalCounterValue" : 2400,
  731. "name" : "SPILLED_RECORDS"
  732. },
  733. {
  734. "reduceCounterValue" : 0,
  735. "mapCounterValue" : 0,
  736. "totalCounterValue" : 1,
  737. "name" : "SHUFFLED_MAPS"
  738. },
  739. {
  740. "reduceCounterValue" : 0,
  741. "mapCounterValue" : 0,
  742. "totalCounterValue" : 0,
  743. "name" : "FAILED_SHUFFLE"
  744. },
  745. {
  746. "reduceCounterValue" : 0,
  747. "mapCounterValue" : 0,
  748. "totalCounterValue" : 1,
  749. "name" : "MERGED_MAP_OUTPUTS"
  750. },
  751. {
  752. "reduceCounterValue" : 0,
  753. "mapCounterValue" : 0,
  754. "totalCounterValue" : 113,
  755. "name" : "GC_TIME_MILLIS"
  756. },
  757. {
  758. "reduceCounterValue" : 0,
  759. "mapCounterValue" : 0,
  760. "totalCounterValue" : 1830,
  761. "name" : "CPU_MILLISECONDS"
  762. },
  763. {
  764. "reduceCounterValue" : 0,
  765. "mapCounterValue" : 0,
  766. "totalCounterValue" : 478068736,
  767. "name" : "PHYSICAL_MEMORY_BYTES"
  768. },
  769. {
  770. "reduceCounterValue" : 0,
  771. "mapCounterValue" : 0,
  772. "totalCounterValue" : 2159284224,
  773. "name" : "VIRTUAL_MEMORY_BYTES"
  774. },
  775. {
  776. "reduceCounterValue" : 0,
  777. "mapCounterValue" : 0,
  778. "totalCounterValue" : 378863616,
  779. "name" : "COMMITTED_HEAP_BYTES"
  780. }
  781. ]
  782. },
  783. {
  784. "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
  785. "counter" : [
  786. {
  787. "reduceCounterValue" : 0,
  788. "mapCounterValue" : 0,
  789. "totalCounterValue" : 0,
  790. "name" : "BYTES_READ"
  791. }
  792. ]
  793. },
  794. {
  795. "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
  796. "counter" : [
  797. {
  798. "reduceCounterValue" : 0,
  799. "mapCounterValue" : 0,
  800. "totalCounterValue" : 0,
  801. "name" : "BYTES_WRITTEN"
  802. }
  803. ]
  804. }
  805. ]
  806. }
  807. }
  808. +---+
  809. <<XML response>>
  810. HTTP Request:
  811. ------
  812. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
  813. Accept: application/xml
  814. ------
  815. Response Header:
  816. +---+
  817. HTTP/1.1 200 OK
  818. Content-Type: application/xml
  819. Content-Length: 7030
  820. Server: Jetty(6.1.26)
  821. +---+
  822. Response Body:
  823. +---+
  824. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  825. <jobCounters>
  826. <id>job_1326381300833_2_2</id>
  827. <counterGroup>
  828. <counterGroupName>Shuffle Errors</counterGroupName>
  829. <counter>
  830. <name>BAD_ID</name>
  831. <totalCounterValue>0</totalCounterValue>
  832. <mapCounterValue>0</mapCounterValue>
  833. <reduceCounterValue>0</reduceCounterValue>
  834. </counter>
  835. <counter>
  836. <name>CONNECTION</name>
  837. <totalCounterValue>0</totalCounterValue>
  838. <mapCounterValue>0</mapCounterValue>
  839. <reduceCounterValue>0</reduceCounterValue>
  840. </counter>
  841. <counter>
  842. <name>IO_ERROR</name>
  843. <totalCounterValue>0</totalCounterValue>
  844. <mapCounterValue>0</mapCounterValue>
  845. <reduceCounterValue>0</reduceCounterValue>
  846. </counter>
  847. <counter>
  848. <name>WRONG_LENGTH</name>
  849. <totalCounterValue>0</totalCounterValue>
  850. <mapCounterValue>0</mapCounterValue>
  851. <reduceCounterValue>0</reduceCounterValue>
  852. </counter>
  853. <counter>
  854. <name>WRONG_MAP</name>
  855. <totalCounterValue>0</totalCounterValue>
  856. <mapCounterValue>0</mapCounterValue>
  857. <reduceCounterValue>0</reduceCounterValue>
  858. </counter>
  859. <counter>
  860. <name>WRONG_REDUCE</name>
  861. <totalCounterValue>0</totalCounterValue>
  862. <mapCounterValue>0</mapCounterValue>
  863. <reduceCounterValue>0</reduceCounterValue>
  864. </counter>
  865. </counterGroup>
  866. <counterGroup>
  867. <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
  868. <counter>
  869. <name>FILE_BYTES_READ</name>
  870. <totalCounterValue>2483</totalCounterValue>
  871. <mapCounterValue>0</mapCounterValue>
  872. <reduceCounterValue>0</reduceCounterValue>
  873. </counter>
  874. <counter>
  875. <name>FILE_BYTES_WRITTEN</name>
  876. <totalCounterValue>108525</totalCounterValue>
  877. <mapCounterValue>0</mapCounterValue>
  878. <reduceCounterValue>0</reduceCounterValue>
  879. </counter>
  880. <counter>
  881. <name>FILE_READ_OPS</name>
  882. <totalCounterValue>0</totalCounterValue>
  883. <mapCounterValue>0</mapCounterValue>
  884. <reduceCounterValue>0</reduceCounterValue>
  885. </counter>
  886. <counter>
  887. <name>FILE_LARGE_READ_OPS</name>
  888. <totalCounterValue>0</totalCounterValue>
  889. <mapCounterValue>0</mapCounterValue>
  890. <reduceCounterValue>0</reduceCounterValue>
  891. </counter>
  892. <counter>
  893. <name>FILE_WRITE_OPS</name>
  894. <totalCounterValue>0</totalCounterValue>
  895. <mapCounterValue>0</mapCounterValue>
  896. <reduceCounterValue>0</reduceCounterValue>
  897. </counter>
  898. <counter>
  899. <name>HDFS_BYTES_READ</name>
  900. <totalCounterValue>48</totalCounterValue>
  901. <mapCounterValue>0</mapCounterValue>
  902. <reduceCounterValue>0</reduceCounterValue>
  903. </counter>
  904. <counter>
  905. <name>HDFS_BYTES_WRITTEN</name>
  906. <totalCounterValue>0</totalCounterValue>
  907. <mapCounterValue>0</mapCounterValue>
  908. <reduceCounterValue>0</reduceCounterValue>
  909. </counter>
  910. <counter>
  911. <name>HDFS_READ_OPS</name>
  912. <totalCounterValue>1</totalCounterValue>
  913. <mapCounterValue>0</mapCounterValue>
  914. <reduceCounterValue>0</reduceCounterValue>
  915. </counter>
  916. <counter>
  917. <name>HDFS_LARGE_READ_OPS</name>
  918. <totalCounterValue>0</totalCounterValue>
  919. <mapCounterValue>0</mapCounterValue>
  920. <reduceCounterValue>0</reduceCounterValue>
  921. </counter>
  922. <counter>
  923. <name>HDFS_WRITE_OPS</name>
  924. <totalCounterValue>0</totalCounterValue>
  925. <mapCounterValue>0</mapCounterValue>
  926. <reduceCounterValue>0</reduceCounterValue>
  927. </counter>
  928. </counterGroup>
  929. <counterGroup>
  930. <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
  931. <counter>
  932. <name>MAP_INPUT_RECORDS</name>
  933. <totalCounterValue>1</totalCounterValue>
  934. <mapCounterValue>0</mapCounterValue>
  935. <reduceCounterValue>0</reduceCounterValue>
  936. </counter>
  937. <counter>
  938. <name>MAP_OUTPUT_RECORDS</name>
  939. <totalCounterValue>1200</totalCounterValue>
  940. <mapCounterValue>0</mapCounterValue>
  941. <reduceCounterValue>0</reduceCounterValue>
  942. </counter>
  943. <counter>
  944. <name>MAP_OUTPUT_BYTES</name>
  945. <totalCounterValue>4800</totalCounterValue>
  946. <mapCounterValue>0</mapCounterValue>
  947. <reduceCounterValue>0</reduceCounterValue>
  948. </counter>
  949. <counter>
  950. <name>MAP_OUTPUT_MATERIALIZED_BYTES</name>
  951. <totalCounterValue>2235</totalCounterValue>
  952. <mapCounterValue>0</mapCounterValue>
  953. <reduceCounterValue>0</reduceCounterValue>
  954. </counter>
  955. <counter>
  956. <name>SPLIT_RAW_BYTES</name>
  957. <totalCounterValue>48</totalCounterValue>
  958. <mapCounterValue>0</mapCounterValue>
  959. <reduceCounterValue>0</reduceCounterValue>
  960. </counter>
  961. <counter>
  962. <name>COMBINE_INPUT_RECORDS</name>
  963. <totalCounterValue>0</totalCounterValue>
  964. <mapCounterValue>0</mapCounterValue>
  965. <reduceCounterValue>0</reduceCounterValue>
  966. </counter>
  967. <counter>
  968. <name>COMBINE_OUTPUT_RECORDS</name>
  969. <totalCounterValue>0</totalCounterValue>
  970. <mapCounterValue>0</mapCounterValue>
  971. <reduceCounterValue>0</reduceCounterValue>
  972. </counter>
  973. <counter>
  974. <name>REDUCE_INPUT_GROUPS</name>
  975. <totalCounterValue>1200</totalCounterValue>
  976. <mapCounterValue>0</mapCounterValue>
  977. <reduceCounterValue>0</reduceCounterValue>
  978. </counter>
  979. <counter>
  980. <name>REDUCE_SHUFFLE_BYTES</name>
  981. <totalCounterValue>2235</totalCounterValue>
  982. <mapCounterValue>0</mapCounterValue>
  983. <reduceCounterValue>0</reduceCounterValue>
  984. </counter>
  985. <counter>
  986. <name>REDUCE_INPUT_RECORDS</name>
  987. <totalCounterValue>1200</totalCounterValue>
  988. <mapCounterValue>0</mapCounterValue>
  989. <reduceCounterValue>0</reduceCounterValue>
  990. </counter>
  991. <counter>
  992. <name>REDUCE_OUTPUT_RECORDS</name>
  993. <totalCounterValue>0</totalCounterValue>
  994. <mapCounterValue>0</mapCounterValue>
  995. <reduceCounterValue>0</reduceCounterValue>
  996. </counter>
  997. <counter>
  998. <name>SPILLED_RECORDS</name>
  999. <totalCounterValue>2400</totalCounterValue>
  1000. <mapCounterValue>0</mapCounterValue>
  1001. <reduceCounterValue>0</reduceCounterValue>
  1002. </counter>
  1003. <counter>
  1004. <name>SHUFFLED_MAPS</name>
  1005. <totalCounterValue>1</totalCounterValue>
  1006. <mapCounterValue>0</mapCounterValue>
  1007. <reduceCounterValue>0</reduceCounterValue>
  1008. </counter>
  1009. <counter>
  1010. <name>FAILED_SHUFFLE</name>
  1011. <totalCounterValue>0</totalCounterValue>
  1012. <mapCounterValue>0</mapCounterValue>
  1013. <reduceCounterValue>0</reduceCounterValue>
  1014. </counter>
  1015. <counter>
  1016. <name>MERGED_MAP_OUTPUTS</name>
  1017. <totalCounterValue>1</totalCounterValue>
  1018. <mapCounterValue>0</mapCounterValue>
  1019. <reduceCounterValue>0</reduceCounterValue>
  1020. </counter>
  1021. <counter>
  1022. <name>GC_TIME_MILLIS</name>
  1023. <totalCounterValue>113</totalCounterValue>
  1024. <mapCounterValue>0</mapCounterValue>
  1025. <reduceCounterValue>0</reduceCounterValue>
  1026. </counter>
  1027. <counter>
  1028. <name>CPU_MILLISECONDS</name>
  1029. <totalCounterValue>1830</totalCounterValue>
  1030. <mapCounterValue>0</mapCounterValue>
  1031. <reduceCounterValue>0</reduceCounterValue>
  1032. </counter>
  1033. <counter>
  1034. <name>PHYSICAL_MEMORY_BYTES</name>
  1035. <totalCounterValue>478068736</totalCounterValue>
  1036. <mapCounterValue>0</mapCounterValue>
  1037. <reduceCounterValue>0</reduceCounterValue>
  1038. </counter>
  1039. <counter>
  1040. <name>VIRTUAL_MEMORY_BYTES</name>
  1041. <totalCounterValue>2159284224</totalCounterValue>
  1042. <mapCounterValue>0</mapCounterValue>
  1043. <reduceCounterValue>0</reduceCounterValue>
  1044. </counter>
  1045. <counter>
  1046. <name>COMMITTED_HEAP_BYTES</name>
  1047. <totalCounterValue>378863616</totalCounterValue>
  1048. <mapCounterValue>0</mapCounterValue>
  1049. <reduceCounterValue>0</reduceCounterValue>
  1050. </counter>
  1051. </counterGroup>
  1052. <counterGroup>
  1053. <counterGroupName>org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter</counterGroupName>
  1054. <counter>
  1055. <name>BYTES_READ</name>
  1056. <totalCounterValue>0</totalCounterValue>
  1057. <mapCounterValue>0</mapCounterValue>
  1058. <reduceCounterValue>0</reduceCounterValue>
  1059. </counter>
  1060. </counterGroup>
  1061. <counterGroup>
  1062. <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
  1063. <counter>
  1064. <name>BYTES_WRITTEN</name>
  1065. <totalCounterValue>0</totalCounterValue>
  1066. <mapCounterValue>0</mapCounterValue>
  1067. <reduceCounterValue>0</reduceCounterValue>
  1068. </counter>
  1069. </counterGroup>
  1070. </jobCounters>
  1071. +---+
  1072. ** Job Conf API
  1073. A job configuration resource contains information about the job configuration for this job.
  1074. *** URI
  1075. Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value.
  1076. ------
  1077. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/conf
  1078. ------
  1079. *** HTTP Operations Supported
  1080. ------
  1081. * GET
  1082. ------
  1083. *** Query Parameters Supported
  1084. ------
  1085. None
  1086. ------
  1087. *** Elements of the <conf> object
  1088. *---------------+--------------+-------------------------------+
  1089. || Item || Data Type || Description |
  1090. *---------------+--------------+-------------------------------+
  1091. | path | string | The path to the job configuration file|
  1092. *---------------+--------------+-------------------------------+
  1093. | property | array of the configuration properties(JSON)/zero or more configuration properties(XML) | Collection of configuration property objects|
  1094. *---------------+--------------+-------------------------------+
  1095. *** Elements of the <property> object
  1096. *---------------+--------------+-------------------------------+
  1097. || Item || Data Type || Description |
  1098. *---------------+--------------+-------------------------------+
  1099. | name | string | The name of the configuration property |
  1100. *---------------+--------------+-------------------------------+
  1101. | value | string | The value of the configuration property |
  1102. *---------------+--------------+-------------------------------+
  1103. *** Response Examples
  1104. <<JSON response>>
  1105. HTTP Request:
  1106. ------
  1107. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
  1108. ------
  1109. Response Header:
  1110. +---+
  1111. HTTP/1.1 200 OK
  1112. Content-Type: application/json
  1113. Transfer-Encoding: chunked
  1114. Server: Jetty(6.1.26)
  1115. +---+
  1116. Response Body:
  1117. This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file.
  1118. +---+
  1119. {
  1120. "conf" : {
  1121. "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml",
  1122. "property" : [
  1123. {
  1124. "value" : "/home/hadoop/hdfs/data",
  1125. "name" : "dfs.datanode.data.dir"
  1126. },
  1127. {
  1128. "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
  1129. "name" : "hadoop.http.filter.initializers"
  1130. },
  1131. {
  1132. "value" : "/home/hadoop/tmp",
  1133. "name" : "mapreduce.cluster.temp.dir"
  1134. },
  1135. ...
  1136. ]
  1137. }
  1138. }
  1139. +---+
  1140. <<XML response>>
  1141. HTTP Request:
  1142. ------
  1143. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
  1144. Accept: application/xml
  1145. ------
  1146. Response Header:
  1147. +---+
  1148. HTTP/1.1 200 OK
  1149. Content-Type: application/xml
  1150. Content-Length: 552
  1151. Server: Jetty(6.1.26)
  1152. +---+
  1153. Response Body:
  1154. +---+
  1155. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1156. <conf>
  1157. <path>hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml</path>
  1158. <property>
  1159. <name>dfs.datanode.data.dir</name>
  1160. <value>/home/hadoop/hdfs/data</value>
  1161. </property>
  1162. <property>
  1163. <name>hadoop.http.filter.initializers</name>
  1164. <value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
  1165. </property>
  1166. <property>
  1167. <name>mapreduce.cluster.temp.dir</name>
  1168. <value>/home/hadoop/tmp</value>
  1169. </property>
  1170. ...
  1171. </conf>
  1172. +---+
  1173. ** Tasks API
  1174. With the tasks API, you can obtain a collection of resources that represent a task within a job. When you run a GET operation on this resource, you obtain a collection of Task Objects.
  1175. *** URI
  1176. ------
  1177. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks
  1178. ------
  1179. *** HTTP Operations Supported
  1180. ------
  1181. * GET
  1182. ------
  1183. *** Query Parameters Supported
  1184. ------
  1185. * type - type of task, valid values are m or r. m for map task or r for reduce task.
  1186. ------
  1187. *** Elements of the <tasks> object
  1188. When you make a request for the list of tasks , the information will be returned as an array of task objects.
  1189. See also {{Task API}} for syntax of the task object.
  1190. *---------------+--------------+-------------------------------+
  1191. || Item || Data Type || Description |
  1192. *---------------+--------------+-------------------------------+
  1193. | task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects. |
  1194. *---------------+--------------+--------------------------------+
  1195. *** Response Examples
  1196. <<JSON response>>
  1197. HTTP Request:
  1198. ------
  1199. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
  1200. ------
  1201. Response Header:
  1202. +---+
  1203. HTTP/1.1 200 OK
  1204. Content-Type: application/json
  1205. Transfer-Encoding: chunked
  1206. Server: Jetty(6.1.26)
  1207. +---+
  1208. Response Body:
  1209. +---+
  1210. {
  1211. "tasks" : {
  1212. "task" : [
  1213. {
  1214. "progress" : 100,
  1215. "elapsedTime" : 6777,
  1216. "state" : "SUCCEEDED",
  1217. "startTime" : 1326381446541,
  1218. "id" : "task_1326381300833_2_2_m_0",
  1219. "type" : "MAP",
  1220. "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
  1221. "finishTime" : 1326381453318
  1222. },
  1223. {
  1224. "progress" : 100,
  1225. "elapsedTime" : 135559,
  1226. "state" : "SUCCEEDED",
  1227. "startTime" : 1326381446544,
  1228. "id" : "task_1326381300833_2_2_r_0",
  1229. "type" : "REDUCE",
  1230. "successfulAttempt" : "attempt_1326381300833_2_2_r_0_0",
  1231. "finishTime" : 1326381582103
  1232. }
  1233. ]
  1234. }
  1235. }
  1236. +---+
  1237. <<XML response>>
  1238. HTTP Request:
  1239. ------
  1240. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
  1241. Accept: application/xml
  1242. ------
  1243. Response Header:
  1244. +---+
  1245. HTTP/1.1 200 OK
  1246. Content-Type: application/xml
  1247. Content-Length: 653
  1248. Server: Jetty(6.1.26)
  1249. +---+
  1250. Response Body:
  1251. +---+
  1252. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1253. <tasks>
  1254. <task>
  1255. <startTime>1326381446541</startTime>
  1256. <finishTime>1326381453318</finishTime>
  1257. <elapsedTime>6777</elapsedTime>
  1258. <progress>100.0</progress>
  1259. <id>task_1326381300833_2_2_m_0</id>
  1260. <state>SUCCEEDED</state>
  1261. <type>MAP</type>
  1262. <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
  1263. </task>
  1264. <task>
  1265. <startTime>1326381446544</startTime>
  1266. <finishTime>1326381582103</finishTime>
  1267. <elapsedTime>135559</elapsedTime>
  1268. <progress>100.0</progress>
  1269. <id>task_1326381300833_2_2_r_0</id>
  1270. <state>SUCCEEDED</state>
  1271. <type>REDUCE</type>
  1272. <successfulAttempt>attempt_1326381300833_2_2_r_0_0</successfulAttempt>
  1273. </task>
  1274. </tasks>
  1275. +---+
  1276. ** {Task API}
  1277. A Task resource contains information about a particular task within a job.
  1278. *** URI
  1279. Use the following URI to obtain an Task Object, from a task identified by the {taskid} value.
  1280. ------
  1281. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}
  1282. ------
  1283. *** HTTP Operations Supported
  1284. ------
  1285. * GET
  1286. ------
  1287. *** Query Parameters Supported
  1288. ------
  1289. None
  1290. ------
  1291. *** Elements of the <task> object
  1292. *---------------+--------------+-------------------------------+
  1293. || Item || Data Type || Description |
  1294. *---------------+--------------+-------------------------------+
  1295. | id | string | The task id |
  1296. *---------------+--------------+--------------------------------+
  1297. | state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED
  1298. *---------------+--------------+--------------------------------+
  1299. | type | string | The task type - MAP or REDUCE|
  1300. *---------------+--------------+--------------------------------+
  1301. | successfulAttempt | string | The id of the last successful attempt |
  1302. *---------------+--------------+--------------------------------+
  1303. | progress | float | The progress of the task as a percent|
  1304. *---------------+--------------+--------------------------------+
  1305. | startTime | long | The time in which the task started (in ms since epoch)|
  1306. *---------------+--------------+--------------------------------+
  1307. | finishTime | long | The time in which the task finished (in ms since epoch)|
  1308. *---------------+--------------+--------------------------------+
  1309. | elapsedTime | long | The elapsed time since the application started (in ms)|
  1310. *---------------+--------------+--------------------------------+
  1311. *** Response Examples
  1312. <<JSON response>>
  1313. HTTP Request:
  1314. ------
  1315. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
  1316. ------
  1317. Response Header:
  1318. +---+
  1319. HTTP/1.1 200 OK
  1320. Content-Type: application/json
  1321. Transfer-Encoding: chunked
  1322. Server: Jetty(6.1.26)
  1323. +---+
  1324. Response Body:
  1325. +---+
  1326. {
  1327. "task" : {
  1328. "progress" : 100,
  1329. "elapsedTime" : 6777,
  1330. "state" : "SUCCEEDED",
  1331. "startTime" : 1326381446541,
  1332. "id" : "task_1326381300833_2_2_m_0",
  1333. "type" : "MAP",
  1334. "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
  1335. "finishTime" : 1326381453318
  1336. }
  1337. }
  1338. +---+
  1339. <<XML response>>
  1340. HTTP Request:
  1341. ------
  1342. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
  1343. Accept: application/xml
  1344. ------
  1345. Response Header:
  1346. +---+
  1347. HTTP/1.1 200 OK
  1348. Content-Type: application/xml
  1349. Content-Length: 299
  1350. Server: Jetty(6.1.26)
  1351. +---+
  1352. Response Body:
  1353. +---+
  1354. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1355. <task>
  1356. <startTime>1326381446541</startTime>
  1357. <finishTime>1326381453318</finishTime>
  1358. <elapsedTime>6777</elapsedTime>
  1359. <progress>100.0</progress>
  1360. <id>task_1326381300833_2_2_m_0</id>
  1361. <state>SUCCEEDED</state>
  1362. <type>MAP</type>
  1363. <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
  1364. </task>
  1365. +---+
  1366. ** Task Counters API
  1367. With the task counters API, you can object a collection of resources that represent al the counters for that task.
  1368. *** URI
  1369. ------
  1370. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
  1371. ------
  1372. *** HTTP Operations Supported
  1373. ------
  1374. * GET
  1375. ------
  1376. *** Query Parameters Supported
  1377. ------
  1378. None
  1379. ------
  1380. *** Elements of the <jobTaskCounters> object
  1381. *---------------+--------------+-------------------------------+
  1382. || Item || Data Type || Description |
  1383. *---------------+--------------+-------------------------------+
  1384. | id | string | The task id |
  1385. *---------------+--------------+-------------------------------+
  1386. | taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
  1387. *---------------+--------------+-------------------------------+
  1388. *** Elements of the <counterGroup> object
  1389. *---------------+--------------+-------------------------------+
  1390. || Item || Data Type || Description |
  1391. *---------------+--------------+-------------------------------+
  1392. | counterGroupName | string | The name of the counter group |
  1393. *---------------+--------------+-------------------------------+
  1394. | counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
  1395. *---------------+--------------+-------------------------------+
  1396. *** Elements of the <counter> object
  1397. *---------------+--------------+-------------------------------+
  1398. || Item || Data Type || Description |
  1399. *---------------+--------------+-------------------------------+
  1400. | name | string | The name of the counter |
  1401. *---------------+--------------+-------------------------------+
  1402. | value | long | The value of the counter |
  1403. *---------------+--------------+-------------------------------+
  1404. *** Response Examples
  1405. <<JSON response>>
  1406. HTTP Request:
  1407. ------
  1408. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
  1409. ------
  1410. Response Header:
  1411. +---+
  1412. HTTP/1.1 200 OK
  1413. Content-Type: application/json
  1414. Transfer-Encoding: chunked
  1415. Server: Jetty(6.1.26)
  1416. +---+
  1417. Response Body:
  1418. +---+
  1419. {
  1420. "jobTaskCounters" : {
  1421. "id" : "task_1326381300833_2_2_m_0",
  1422. "taskCounterGroup" : [
  1423. {
  1424. "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
  1425. "counter" : [
  1426. {
  1427. "value" : 2363,
  1428. "name" : "FILE_BYTES_READ"
  1429. },
  1430. {
  1431. "value" : 54372,
  1432. "name" : "FILE_BYTES_WRITTEN"
  1433. },
  1434. {
  1435. "value" : 0,
  1436. "name" : "FILE_READ_OPS"
  1437. },
  1438. {
  1439. "value" : 0,
  1440. "name" : "FILE_LARGE_READ_OPS"
  1441. },
  1442. {
  1443. "value" : 0,
  1444. "name" : "FILE_WRITE_OPS"
  1445. },
  1446. {
  1447. "value" : 0,
  1448. "name" : "HDFS_BYTES_READ"
  1449. },
  1450. {
  1451. "value" : 0,
  1452. "name" : "HDFS_BYTES_WRITTEN"
  1453. },
  1454. {
  1455. "value" : 0,
  1456. "name" : "HDFS_READ_OPS"
  1457. },
  1458. {
  1459. "value" : 0,
  1460. "name" : "HDFS_LARGE_READ_OPS"
  1461. },
  1462. {
  1463. "value" : 0,
  1464. "name" : "HDFS_WRITE_OPS"
  1465. }
  1466. ]
  1467. },
  1468. {
  1469. "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
  1470. "counter" : [
  1471. {
  1472. "value" : 0,
  1473. "name" : "COMBINE_INPUT_RECORDS"
  1474. },
  1475. {
  1476. "value" : 0,
  1477. "name" : "COMBINE_OUTPUT_RECORDS"
  1478. },
  1479. {
  1480. "value" : 460,
  1481. "name" : "REDUCE_INPUT_GROUPS"
  1482. },
  1483. {
  1484. "value" : 2235,
  1485. "name" : "REDUCE_SHUFFLE_BYTES"
  1486. },
  1487. {
  1488. "value" : 460,
  1489. "name" : "REDUCE_INPUT_RECORDS"
  1490. },
  1491. {
  1492. "value" : 0,
  1493. "name" : "REDUCE_OUTPUT_RECORDS"
  1494. },
  1495. {
  1496. "value" : 0,
  1497. "name" : "SPILLED_RECORDS"
  1498. },
  1499. {
  1500. "value" : 1,
  1501. "name" : "SHUFFLED_MAPS"
  1502. },
  1503. {
  1504. "value" : 0,
  1505. "name" : "FAILED_SHUFFLE"
  1506. },
  1507. {
  1508. "value" : 1,
  1509. "name" : "MERGED_MAP_OUTPUTS"
  1510. },
  1511. {
  1512. "value" : 26,
  1513. "name" : "GC_TIME_MILLIS"
  1514. },
  1515. {
  1516. "value" : 860,
  1517. "name" : "CPU_MILLISECONDS"
  1518. },
  1519. {
  1520. "value" : 107839488,
  1521. "name" : "PHYSICAL_MEMORY_BYTES"
  1522. },
  1523. {
  1524. "value" : 1123147776,
  1525. "name" : "VIRTUAL_MEMORY_BYTES"
  1526. },
  1527. {
  1528. "value" : 57475072,
  1529. "name" : "COMMITTED_HEAP_BYTES"
  1530. }
  1531. ]
  1532. },
  1533. {
  1534. "counterGroupName" : "Shuffle Errors",
  1535. "counter" : [
  1536. {
  1537. "value" : 0,
  1538. "name" : "BAD_ID"
  1539. },
  1540. {
  1541. "value" : 0,
  1542. "name" : "CONNECTION"
  1543. },
  1544. {
  1545. "value" : 0,
  1546. "name" : "IO_ERROR"
  1547. },
  1548. {
  1549. "value" : 0,
  1550. "name" : "WRONG_LENGTH"
  1551. },
  1552. {
  1553. "value" : 0,
  1554. "name" : "WRONG_MAP"
  1555. },
  1556. {
  1557. "value" : 0,
  1558. "name" : "WRONG_REDUCE"
  1559. }
  1560. ]
  1561. },
  1562. {
  1563. "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
  1564. "counter" : [
  1565. {
  1566. "value" : 0,
  1567. "name" : "BYTES_WRITTEN"
  1568. }
  1569. ]
  1570. }
  1571. ]
  1572. }
  1573. }
  1574. +---+
  1575. <<XML response>>
  1576. HTTP Request:
  1577. ------
  1578. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
  1579. Accept: application/xml
  1580. ------
  1581. Response Header:
  1582. +---+
  1583. HTTP/1.1 200 OK
  1584. Content-Type: application/xml
  1585. Content-Length: 2660
  1586. Server: Jetty(6.1.26)
  1587. +---+
  1588. Response Body:
  1589. +---+
  1590. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1591. <jobTaskCounters>
  1592. <id>task_1326381300833_2_2_m_0</id>
  1593. <taskCounterGroup>
  1594. <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
  1595. <counter>
  1596. <name>FILE_BYTES_READ</name>
  1597. <value>2363</value>
  1598. </counter>
  1599. <counter>
  1600. <name>FILE_BYTES_WRITTEN</name>
  1601. <value>54372</value>
  1602. </counter>
  1603. <counter>
  1604. <name>FILE_READ_OPS</name>
  1605. <value>0</value>
  1606. </counter>
  1607. <counter>
  1608. <name>FILE_LARGE_READ_OPS</name>
  1609. <value>0</value>
  1610. </counter>
  1611. <counter>
  1612. <name>FILE_WRITE_OPS</name>
  1613. <value>0</value>
  1614. </counter>
  1615. <counter>
  1616. <name>HDFS_BYTES_READ</name>
  1617. <value>0</value>
  1618. </counter>
  1619. <counter>
  1620. <name>HDFS_BYTES_WRITTEN</name>
  1621. <value>0</value>
  1622. </counter>
  1623. <counter>
  1624. <name>HDFS_READ_OPS</name>
  1625. <value>0</value>
  1626. </counter>
  1627. <counter>
  1628. <name>HDFS_LARGE_READ_OPS</name>
  1629. <value>0</value>
  1630. </counter>
  1631. <counter>
  1632. <name>HDFS_WRITE_OPS</name>
  1633. <value>0</value>
  1634. </counter>
  1635. </taskCounterGroup>
  1636. <taskCounterGroup>
  1637. <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
  1638. <counter>
  1639. <name>COMBINE_INPUT_RECORDS</name>
  1640. <value>0</value>
  1641. </counter>
  1642. <counter>
  1643. <name>COMBINE_OUTPUT_RECORDS</name>
  1644. <value>0</value>
  1645. </counter>
  1646. <counter>
  1647. <name>REDUCE_INPUT_GROUPS</name>
  1648. <value>460</value>
  1649. </counter>
  1650. <counter>
  1651. <name>REDUCE_SHUFFLE_BYTES</name>
  1652. <value>2235</value>
  1653. </counter>
  1654. <counter>
  1655. <name>REDUCE_INPUT_RECORDS</name>
  1656. <value>460</value>
  1657. </counter>
  1658. <counter>
  1659. <name>REDUCE_OUTPUT_RECORDS</name>
  1660. <value>0</value>
  1661. </counter>
  1662. <counter>
  1663. <name>SPILLED_RECORDS</name>
  1664. <value>0</value>
  1665. </counter>
  1666. <counter>
  1667. <name>SHUFFLED_MAPS</name>
  1668. <value>1</value>
  1669. </counter>
  1670. <counter>
  1671. <name>FAILED_SHUFFLE</name>
  1672. <value>0</value>
  1673. </counter>
  1674. <counter>
  1675. <name>MERGED_MAP_OUTPUTS</name>
  1676. <value>1</value>
  1677. </counter>
  1678. <counter>
  1679. <name>GC_TIME_MILLIS</name>
  1680. <value>26</value>
  1681. </counter>
  1682. <counter>
  1683. <name>CPU_MILLISECONDS</name>
  1684. <value>860</value>
  1685. </counter>
  1686. <counter>
  1687. <name>PHYSICAL_MEMORY_BYTES</name>
  1688. <value>107839488</value>
  1689. </counter>
  1690. <counter>
  1691. <name>VIRTUAL_MEMORY_BYTES</name>
  1692. <value>1123147776</value>
  1693. </counter>
  1694. <counter>
  1695. <name>COMMITTED_HEAP_BYTES</name>
  1696. <value>57475072</value>
  1697. </counter>
  1698. </taskCounterGroup>
  1699. <taskCounterGroup>
  1700. <counterGroupName>Shuffle Errors</counterGroupName>
  1701. <counter>
  1702. <name>BAD_ID</name>
  1703. <value>0</value>
  1704. </counter>
  1705. <counter>
  1706. <name>CONNECTION</name>
  1707. <value>0</value>
  1708. </counter>
  1709. <counter>
  1710. <name>IO_ERROR</name>
  1711. <value>0</value>
  1712. </counter>
  1713. <counter>
  1714. <name>WRONG_LENGTH</name>
  1715. <value>0</value>
  1716. </counter>
  1717. <counter>
  1718. <name>WRONG_MAP</name>
  1719. <value>0</value>
  1720. </counter>
  1721. <counter>
  1722. <name>WRONG_REDUCE</name>
  1723. <value>0</value>
  1724. </counter>
  1725. </taskCounterGroup>
  1726. <taskCounterGroup>
  1727. <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
  1728. <counter>
  1729. <name>BYTES_WRITTEN</name>
  1730. <value>0</value>
  1731. </counter>
  1732. </taskCounterGroup>
  1733. </jobTaskCounters>
  1734. +---+
  1735. ** Task Attempts API
  1736. With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects.
  1737. *** URI
  1738. ------
  1739. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
  1740. ------
  1741. *** HTTP Operations Supported
  1742. ------
  1743. * GET
  1744. ------
  1745. *** Query Parameters Supported
  1746. ------
  1747. None
  1748. ------
  1749. *** Elements of the <taskAttempts> object
  1750. When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects.
  1751. See also {{Task Attempt API}} for syntax of the task object.
  1752. *---------------+--------------+-------------------------------+
  1753. || Item || Data Type || Description |
  1754. *---------------+--------------+-------------------------------+
  1755. | taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects |
  1756. *---------------+--------------+--------------------------------+
  1757. *** Response Examples
  1758. <<JSON response>>
  1759. HTTP Request:
  1760. ------
  1761. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
  1762. ------
  1763. Response Header:
  1764. +---+
  1765. HTTP/1.1 200 OK
  1766. Content-Type: application/json
  1767. Transfer-Encoding: chunked
  1768. Server: Jetty(6.1.26)
  1769. +---+
  1770. Response Body:
  1771. +---+
  1772. {
  1773. "taskAttempts" : {
  1774. "taskAttempt" : [
  1775. {
  1776. "assignedContainerId" : "container_1326381300833_0002_01_000002",
  1777. "progress" : 100,
  1778. "elapsedTime" : 2638,
  1779. "state" : "SUCCEEDED",
  1780. "diagnostics" : "",
  1781. "rack" : "/98.139.92.0",
  1782. "nodeHttpAddress" : "host.domain.com:8042",
  1783. "startTime" : 1326381450680,
  1784. "id" : "attempt_1326381300833_2_2_m_0_0",
  1785. "type" : "MAP",
  1786. "finishTime" : 1326381453318
  1787. }
  1788. ]
  1789. }
  1790. }
  1791. +---+
  1792. <<XML response>>
  1793. HTTP Request:
  1794. ------
  1795. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
  1796. Accept: application/xml
  1797. ------
  1798. Response Header:
  1799. +---+
  1800. HTTP/1.1 200 OK
  1801. Content-Type: application/xml
  1802. Content-Length: 537
  1803. Server: Jetty(6.1.26)
  1804. +---+
  1805. Response Body:
  1806. +---+
  1807. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1808. <taskAttempts>
  1809. <taskAttempt>
  1810. <startTime>1326381450680</startTime>
  1811. <finishTime>1326381453318</finishTime>
  1812. <elapsedTime>2638</elapsedTime>
  1813. <progress>100.0</progress>
  1814. <id>attempt_1326381300833_2_2_m_0_0</id>
  1815. <rack>/98.139.92.0</rack>
  1816. <state>SUCCEEDED</state>
  1817. <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
  1818. <diagnostics/>
  1819. <type>MAP</type>
  1820. <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
  1821. </taskAttempt>
  1822. </taskAttempts>
  1823. +---+
  1824. ** {Task Attempt API}
  1825. A Task Attempt resource contains information about a particular task attempt within a job.
  1826. *** URI
  1827. Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value.
  1828. ------
  1829. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}
  1830. ------
  1831. *** HTTP Operations Supported
  1832. ------
  1833. * GET
  1834. ------
  1835. *** Query Parameters Supported
  1836. ------
  1837. None
  1838. ------
  1839. *** Elements of the <taskAttempt> object
  1840. *---------------+--------------+-------------------------------+
  1841. || Item || Data Type || Description |
  1842. *---------------+--------------+-------------------------------+
  1843. | id | string | The task id |
  1844. *---------------+--------------+--------------------------------+
  1845. | rack | string | The rack |
  1846. *---------------+--------------+--------------------------------+
  1847. | state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED |
  1848. *---------------+--------------+--------------------------------+
  1849. | type | string | The type of task |
  1850. *---------------+--------------+--------------------------------+
  1851. | assignedContainerId | string | The container id this attempt is assigned to|
  1852. *---------------+--------------+--------------------------------+
  1853. | nodeHttpAddress | string | The http address of the node this task attempt ran on |
  1854. *---------------+--------------+--------------------------------+
  1855. | diagnostics| string | A diagnostics message |
  1856. *---------------+--------------+--------------------------------+
  1857. | progress | float | The progress of the task attempt as a percent|
  1858. *---------------+--------------+--------------------------------+
  1859. | startTime | long | The time in which the task attempt started (in ms since epoch)|
  1860. *---------------+--------------+--------------------------------+
  1861. | finishTime | long | The time in which the task attempt finished (in ms since epoch)|
  1862. *---------------+--------------+--------------------------------+
  1863. | elapsedTime | long | The elapsed time since the task attempt started (in ms)|
  1864. *---------------+--------------+--------------------------------+
  1865. For reduce task attempts you also have the following fields:
  1866. *---------------+--------------+-------------------------------+
  1867. || Item || Data Type || Description |
  1868. *---------------+--------------+-------------------------------+
  1869. | shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)|
  1870. *---------------+--------------+--------------------------------+
  1871. | mergeFinishTime | long | The time at which merge finished (in ms since epoch)|
  1872. *---------------+--------------+--------------------------------+
  1873. | elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)|
  1874. *---------------+--------------+--------------------------------+
  1875. | elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)|
  1876. *---------------+--------------+--------------------------------+
  1877. | elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)|
  1878. *---------------+--------------+--------------------------------+
  1879. *** Response Examples
  1880. <<JSON response>>
  1881. HTTP Request:
  1882. ------
  1883. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
  1884. ------
  1885. Response Header:
  1886. +---+
  1887. HTTP/1.1 200 OK
  1888. Content-Type: application/json
  1889. Transfer-Encoding: chunked
  1890. Server: Jetty(6.1.26)
  1891. +---+
  1892. Response Body:
  1893. +---+
  1894. {
  1895. "taskAttempt" : {
  1896. "assignedContainerId" : "container_1326381300833_0002_01_000002",
  1897. "progress" : 100,
  1898. "elapsedTime" : 2638,
  1899. "state" : "SUCCEEDED",
  1900. "diagnostics" : "",
  1901. "rack" : "/98.139.92.0",
  1902. "nodeHttpAddress" : "host.domain.com:8042",
  1903. "startTime" : 1326381450680,
  1904. "id" : "attempt_1326381300833_2_2_m_0_0",
  1905. "type" : "MAP",
  1906. "finishTime" : 1326381453318
  1907. }
  1908. }
  1909. +---+
  1910. <<XML response>>
  1911. HTTP Request:
  1912. ------
  1913. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
  1914. Accept: application/xml
  1915. ------
  1916. Response Header:
  1917. +---+
  1918. HTTP/1.1 200 OK
  1919. Content-Type: application/xml
  1920. Content-Length: 691
  1921. Server: Jetty(6.1.26)
  1922. +---+
  1923. Response Body:
  1924. +---+
  1925. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  1926. <taskAttempt>
  1927. <startTime>1326381450680</startTime>
  1928. <finishTime>1326381453318</finishTime>
  1929. <elapsedTime>2638</elapsedTime>
  1930. <progress>100.0</progress>
  1931. <id>attempt_1326381300833_2_2_m_0_0</id>
  1932. <rack>/98.139.92.0</rack>
  1933. <state>SUCCEEDED</state>
  1934. <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
  1935. <diagnostics/>
  1936. <type>MAP</type>
  1937. <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
  1938. </taskAttempt>
  1939. +---+
  1940. ** Task Attempt Counters API
  1941. With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt.
  1942. *** URI
  1943. ------
  1944. * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters
  1945. ------
  1946. *** HTTP Operations Supported
  1947. ------
  1948. * GET
  1949. ------
  1950. *** Query Parameters Supported
  1951. ------
  1952. None
  1953. ------
  1954. *** Elements of the <jobTaskAttemptCounters> object
  1955. *---------------+--------------+-------------------------------+
  1956. || Item || Data Type || Description |
  1957. *---------------+--------------+-------------------------------+
  1958. | id | string | The task attempt id |
  1959. *---------------+--------------+-------------------------------+
  1960. | taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects |
  1961. *---------------+--------------+-------------------------------+
  1962. *** Elements of the <taskAttemptCounterGroup> object
  1963. *---------------+--------------+-------------------------------+
  1964. || Item || Data Type || Description |
  1965. *---------------+--------------+-------------------------------+
  1966. | counterGroupName | string | The name of the counter group |
  1967. *---------------+--------------+-------------------------------+
  1968. | counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
  1969. *---------------+--------------+-------------------------------+
  1970. *** Elements of the <counter> object
  1971. *---------------+--------------+-------------------------------+
  1972. || Item || Data Type || Description |
  1973. *---------------+--------------+-------------------------------+
  1974. | name | string | The name of the counter |
  1975. *---------------+--------------+-------------------------------+
  1976. | value | long | The value of the counter |
  1977. *---------------+--------------+-------------------------------+
  1978. *** Response Examples
  1979. <<JSON response>>
  1980. HTTP Request:
  1981. ------
  1982. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
  1983. ------
  1984. Response Header:
  1985. +---+
  1986. HTTP/1.1 200 OK
  1987. Content-Type: application/json
  1988. Transfer-Encoding: chunked
  1989. Server: Jetty(6.1.26)
  1990. +---+
  1991. Response Body:
  1992. +---+
  1993. {
  1994. "jobTaskAttemptCounters" : {
  1995. "taskAttemptCounterGroup" : [
  1996. {
  1997. "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
  1998. "counter" : [
  1999. {
  2000. "value" : 2363,
  2001. "name" : "FILE_BYTES_READ"
  2002. },
  2003. {
  2004. "value" : 54372,
  2005. "name" : "FILE_BYTES_WRITTEN"
  2006. },
  2007. {
  2008. "value" : 0,
  2009. "name" : "FILE_READ_OPS"
  2010. },
  2011. {
  2012. "value" : 0,
  2013. "name" : "FILE_LARGE_READ_OPS"
  2014. },
  2015. {
  2016. "value" : 0,
  2017. "name" : "FILE_WRITE_OPS"
  2018. },
  2019. {
  2020. "value" : 0,
  2021. "name" : "HDFS_BYTES_READ"
  2022. },
  2023. {
  2024. "value" : 0,
  2025. "name" : "HDFS_BYTES_WRITTEN"
  2026. },
  2027. {
  2028. "value" : 0,
  2029. "name" : "HDFS_READ_OPS"
  2030. },
  2031. {
  2032. "value" : 0,
  2033. "name" : "HDFS_LARGE_READ_OPS"
  2034. },
  2035. {
  2036. "value" : 0,
  2037. "name" : "HDFS_WRITE_OPS"
  2038. }
  2039. ]
  2040. },
  2041. {
  2042. "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
  2043. "counter" : [
  2044. {
  2045. "value" : 0,
  2046. "name" : "COMBINE_INPUT_RECORDS"
  2047. },
  2048. {
  2049. "value" : 0,
  2050. "name" : "COMBINE_OUTPUT_RECORDS"
  2051. },
  2052. {
  2053. "value" : 460,
  2054. "name" : "REDUCE_INPUT_GROUPS"
  2055. },
  2056. {
  2057. "value" : 2235,
  2058. "name" : "REDUCE_SHUFFLE_BYTES"
  2059. },
  2060. {
  2061. "value" : 460,
  2062. "name" : "REDUCE_INPUT_RECORDS"
  2063. },
  2064. {
  2065. "value" : 0,
  2066. "name" : "REDUCE_OUTPUT_RECORDS"
  2067. },
  2068. {
  2069. "value" : 0,
  2070. "name" : "SPILLED_RECORDS"
  2071. },
  2072. {
  2073. "value" : 1,
  2074. "name" : "SHUFFLED_MAPS"
  2075. },
  2076. {
  2077. "value" : 0,
  2078. "name" : "FAILED_SHUFFLE"
  2079. },
  2080. {
  2081. "value" : 1,
  2082. "name" : "MERGED_MAP_OUTPUTS"
  2083. },
  2084. {
  2085. "value" : 26,
  2086. "name" : "GC_TIME_MILLIS"
  2087. },
  2088. {
  2089. "value" : 860,
  2090. "name" : "CPU_MILLISECONDS"
  2091. },
  2092. {
  2093. "value" : 107839488,
  2094. "name" : "PHYSICAL_MEMORY_BYTES"
  2095. },
  2096. {
  2097. "value" : 1123147776,
  2098. "name" : "VIRTUAL_MEMORY_BYTES"
  2099. },
  2100. {
  2101. "value" : 57475072,
  2102. "name" : "COMMITTED_HEAP_BYTES"
  2103. }
  2104. ]
  2105. },
  2106. {
  2107. "counterGroupName" : "Shuffle Errors",
  2108. "counter" : [
  2109. {
  2110. "value" : 0,
  2111. "name" : "BAD_ID"
  2112. },
  2113. {
  2114. "value" : 0,
  2115. "name" : "CONNECTION"
  2116. },
  2117. {
  2118. "value" : 0,
  2119. "name" : "IO_ERROR"
  2120. },
  2121. {
  2122. "value" : 0,
  2123. "name" : "WRONG_LENGTH"
  2124. },
  2125. {
  2126. "value" : 0,
  2127. "name" : "WRONG_MAP"
  2128. },
  2129. {
  2130. "value" : 0,
  2131. "name" : "WRONG_REDUCE"
  2132. }
  2133. ]
  2134. },
  2135. {
  2136. "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
  2137. "counter" : [
  2138. {
  2139. "value" : 0,
  2140. "name" : "BYTES_WRITTEN"
  2141. }
  2142. ]
  2143. }
  2144. ],
  2145. "id" : "attempt_1326381300833_2_2_m_0_0"
  2146. }
  2147. }
  2148. +---+
  2149. <<XML response>>
  2150. HTTP Request:
  2151. ------
  2152. GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
  2153. Accept: application/xml
  2154. ------
  2155. Response Header:
  2156. +---+
  2157. HTTP/1.1 200 OK
  2158. Content-Type: application/xml
  2159. Content-Length: 2735
  2160. Server: Jetty(6.1.26)
  2161. +---+
  2162. Response Body:
  2163. +---+
  2164. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  2165. <jobTaskAttemptCounters>
  2166. <id>attempt_1326381300833_2_2_m_0_0</id>
  2167. <taskAttemptCounterGroup>
  2168. <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
  2169. <counter>
  2170. <name>FILE_BYTES_READ</name>
  2171. <value>2363</value>
  2172. </counter>
  2173. <counter>
  2174. <name>FILE_BYTES_WRITTEN</name>
  2175. <value>54372</value>
  2176. </counter>
  2177. <counter>
  2178. <name>FILE_READ_OPS</name>
  2179. <value>0</value>
  2180. </counter>
  2181. <counter>
  2182. <name>FILE_LARGE_READ_OPS</name>
  2183. <value>0</value>
  2184. </counter>
  2185. <counter>
  2186. <name>FILE_WRITE_OPS</name>
  2187. <value>0</value>
  2188. </counter>
  2189. <counter>
  2190. <name>HDFS_BYTES_READ</name>
  2191. <value>0</value>
  2192. </counter>
  2193. <counter>
  2194. <name>HDFS_BYTES_WRITTEN</name>
  2195. <value>0</value>
  2196. </counter>
  2197. <counter>
  2198. <name>HDFS_READ_OPS</name>
  2199. <value>0</value>
  2200. </counter>
  2201. <counter>
  2202. <name>HDFS_LARGE_READ_OPS</name>
  2203. <value>0</value>
  2204. </counter>
  2205. <counter>
  2206. <name>HDFS_WRITE_OPS</name>
  2207. <value>0</value>
  2208. </counter>
  2209. </taskAttemptCounterGroup>
  2210. <taskAttemptCounterGroup>
  2211. <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
  2212. <counter>
  2213. <name>COMBINE_INPUT_RECORDS</name>
  2214. <value>0</value>
  2215. </counter>
  2216. <counter>
  2217. <name>COMBINE_OUTPUT_RECORDS</name>
  2218. <value>0</value>
  2219. </counter>
  2220. <counter>
  2221. <name>REDUCE_INPUT_GROUPS</name>
  2222. <value>460</value>
  2223. </counter>
  2224. <counter>
  2225. <name>REDUCE_SHUFFLE_BYTES</name>
  2226. <value>2235</value>
  2227. </counter>
  2228. <counter>
  2229. <name>REDUCE_INPUT_RECORDS</name>
  2230. <value>460</value>
  2231. </counter>
  2232. <counter>
  2233. <name>REDUCE_OUTPUT_RECORDS</name>
  2234. <value>0</value>
  2235. </counter>
  2236. <counter>
  2237. <name>SPILLED_RECORDS</name>
  2238. <value>0</value>
  2239. </counter>
  2240. <counter>
  2241. <name>SHUFFLED_MAPS</name>
  2242. <value>1</value>
  2243. </counter>
  2244. <counter>
  2245. <name>FAILED_SHUFFLE</name>
  2246. <value>0</value>
  2247. </counter>
  2248. <counter>
  2249. <name>MERGED_MAP_OUTPUTS</name>
  2250. <value>1</value>
  2251. </counter>
  2252. <counter>
  2253. <name>GC_TIME_MILLIS</name>
  2254. <value>26</value>
  2255. </counter>
  2256. <counter>
  2257. <name>CPU_MILLISECONDS</name>
  2258. <value>860</value>
  2259. </counter>
  2260. <counter>
  2261. <name>PHYSICAL_MEMORY_BYTES</name>
  2262. <value>107839488</value>
  2263. </counter>
  2264. <counter>
  2265. <name>VIRTUAL_MEMORY_BYTES</name>
  2266. <value>1123147776</value>
  2267. </counter>
  2268. <counter>
  2269. <name>COMMITTED_HEAP_BYTES</name>
  2270. <value>57475072</value>
  2271. </counter>
  2272. </taskAttemptCounterGroup>
  2273. <taskAttemptCounterGroup>
  2274. <counterGroupName>Shuffle Errors</counterGroupName>
  2275. <counter>
  2276. <name>BAD_ID</name>
  2277. <value>0</value>
  2278. </counter>
  2279. <counter>
  2280. <name>CONNECTION</name>
  2281. <value>0</value>
  2282. </counter>
  2283. <counter>
  2284. <name>IO_ERROR</name>
  2285. <value>0</value>
  2286. </counter>
  2287. <counter>
  2288. <name>WRONG_LENGTH</name>
  2289. <value>0</value>
  2290. </counter>
  2291. <counter>
  2292. <name>WRONG_MAP</name>
  2293. <value>0</value>
  2294. </counter>
  2295. <counter>
  2296. <name>WRONG_REDUCE</name>
  2297. <value>0</value>
  2298. </counter>
  2299. </taskAttemptCounterGroup>
  2300. <taskAttemptCounterGroup>
  2301. <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
  2302. <counter>
  2303. <name>BYTES_WRITTEN</name>
  2304. <value>0</value>
  2305. </counter>
  2306. </taskAttemptCounterGroup>
  2307. </jobTaskAttemptCounters>
  2308. +---+