YARN-9615-branch-3.3-001.patch.1 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732
  1. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
  2. index f9deab06ff2..667515d00c1 100644
  3. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
  4. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
  5. @@ -26,6 +26,9 @@
  6. import java.util.concurrent.BlockingQueue;
  7. import java.util.concurrent.LinkedBlockingQueue;
  8. +import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
  9. +import org.apache.hadoop.yarn.util.Clock;
  10. +import org.apache.hadoop.yarn.util.MonotonicClock;
  11. import org.slf4j.Logger;
  12. import org.slf4j.LoggerFactory;
  13. import org.slf4j.Marker;
  14. @@ -85,6 +88,11 @@
  15. protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
  16. private boolean exitOnDispatchException = true;
  17. + private Map<Class<? extends Enum>,
  18. + EventTypeMetrics> eventTypeMetricsMap;
  19. +
  20. + private Clock clock = new MonotonicClock();
  21. +
  22. /**
  23. * The thread name for dispatcher.
  24. */
  25. @@ -98,6 +106,8 @@ public AsyncDispatcher(BlockingQueue<Event> eventQueue) {
  26. super("Dispatcher");
  27. this.eventQueue = eventQueue;
  28. this.eventDispatchers = new HashMap<Class<? extends Enum>, EventHandler>();
  29. + this.eventTypeMetricsMap = new HashMap<Class<? extends Enum>,
  30. + EventTypeMetrics>();
  31. }
  32. /**
  33. @@ -135,7 +145,16 @@ public void run() {
  34. return;
  35. }
  36. if (event != null) {
  37. - dispatch(event);
  38. + if (eventTypeMetricsMap.
  39. + get(event.getType().getDeclaringClass()) != null) {
  40. + long startTime = clock.getTime();
  41. + dispatch(event);
  42. + eventTypeMetricsMap.get(event.getType().getDeclaringClass())
  43. + .increment(event.getType(),
  44. + clock.getTime() - startTime);
  45. + } else {
  46. + dispatch(event);
  47. + }
  48. if (printTrigger) {
  49. //Log the latest dispatch event type
  50. // may cause the too many events queued
  51. @@ -369,4 +388,9 @@ protected boolean isDrained() {
  52. protected boolean isStopped() {
  53. return stopped;
  54. }
  55. +
  56. + public void addMetrics(EventTypeMetrics metrics,
  57. + Class<? extends Enum> eventClass) {
  58. + eventTypeMetricsMap.put(eventClass, metrics);
  59. + }
  60. }
  61. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
  62. index cadb73663a0..849bb402d87 100644
  63. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
  64. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
  65. @@ -19,6 +19,9 @@
  66. package org.apache.hadoop.yarn.event;
  67. import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
  68. +import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
  69. +import org.apache.hadoop.yarn.util.Clock;
  70. +import org.apache.hadoop.yarn.util.MonotonicClock;
  71. import org.slf4j.Logger;
  72. import org.slf4j.LoggerFactory;
  73. import org.slf4j.Marker;
  74. @@ -47,12 +50,15 @@
  75. private final Thread eventProcessor;
  76. private volatile boolean stopped = false;
  77. private boolean shouldExitOnError = true;
  78. + private EventTypeMetrics metrics;
  79. private static final Logger LOG =
  80. LoggerFactory.getLogger(EventDispatcher.class);
  81. private static final Marker FATAL =
  82. MarkerFactory.getMarker("FATAL");
  83. + private Clock clock = new MonotonicClock();
  84. +
  85. private final class EventProcessor implements Runnable {
  86. @Override
  87. public void run() {
  88. @@ -68,7 +74,14 @@ public void run() {
  89. }
  90. try {
  91. - handler.handle(event);
  92. + if (metrics != null) {
  93. + long startTime = clock.getTime();
  94. + handler.handle(event);
  95. + metrics.increment(event.getType(),
  96. + clock.getTime() - startTime);
  97. + } else {
  98. + handler.handle(event);
  99. + }
  100. } catch (Throwable t) {
  101. // An error occurred, but we are shutting down anyway.
  102. // If it was an InterruptedException, the very act of
  103. @@ -136,6 +149,7 @@ public void handle(T event) {
  104. public void disableExitOnError() {
  105. shouldExitOnError = false;
  106. }
  107. +
  108. protected long getEventProcessorId() {
  109. return this.eventProcessor.getId();
  110. }
  111. @@ -143,4 +157,9 @@ protected long getEventProcessorId() {
  112. protected boolean isStopped() {
  113. return this.stopped;
  114. }
  115. +
  116. + public void setMetrics(EventTypeMetrics metrics) {
  117. + this.metrics = metrics;
  118. + }
  119. +
  120. }
  121. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java
  122. new file mode 100644
  123. index 00000000000..7b4af0c3e09
  124. --- /dev/null
  125. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java
  126. @@ -0,0 +1,42 @@
  127. +/**
  128. + * Licensed to the Apache Software Foundation (ASF) under one
  129. + * or more contributor license agreements. See the NOTICE file
  130. + * distributed with this work for additional information
  131. + * regarding copyright ownership. The ASF licenses this file
  132. + * to you under the Apache License, Version 2.0 (the
  133. + * "License"); you may not use this file except in compliance
  134. + * with the License. You may obtain a copy of the License at
  135. + *
  136. + * http://www.apache.org/licenses/LICENSE-2.0
  137. + *
  138. + * Unless required by applicable law or agreed to in writing, software
  139. + * distributed under the License is distributed on an "AS IS" BASIS,
  140. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  141. + * See the License for the specific language governing permissions and
  142. + * limitations under the License.
  143. + */
  144. +package org.apache.hadoop.yarn.metrics;
  145. +
  146. +import org.apache.hadoop.classification.InterfaceAudience;
  147. +import org.apache.hadoop.metrics2.MetricsCollector;
  148. +import org.apache.hadoop.metrics2.annotation.Metrics;
  149. +
  150. +@InterfaceAudience.Private
  151. +@Metrics(context="yarn")
  152. +public class DisableEventTypeMetrics implements EventTypeMetrics {
  153. + @Override
  154. + public void increment(Enum type, long processingTimeUs) {
  155. + //nop
  156. + return;
  157. + }
  158. + @Override
  159. + public void getMetrics(MetricsCollector collector, boolean all) {
  160. + //nop
  161. + return;
  162. + }
  163. +
  164. + @Override
  165. + public long get(Enum type) {
  166. + return 0;
  167. + }
  168. +}
  169. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/EventTypeMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/EventTypeMetrics.java
  170. new file mode 100644
  171. index 00000000000..7a7e4f54890
  172. --- /dev/null
  173. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/EventTypeMetrics.java
  174. @@ -0,0 +1,32 @@
  175. +/**
  176. + * Licensed to the Apache Software Foundation (ASF) under one
  177. + * or more contributor license agreements. See the NOTICE file
  178. + * distributed with this work for additional information
  179. + * regarding copyright ownership. The ASF licenses this file
  180. + * to you under the Apache License, Version 2.0 (the
  181. + * "License"); you may not use this file except in compliance
  182. + * with the License. You may obtain a copy of the License at
  183. + *
  184. + * http://www.apache.org/licenses/LICENSE-2.0
  185. + *
  186. + * Unless required by applicable law or agreed to in writing, software
  187. + * distributed under the License is distributed on an "AS IS" BASIS,
  188. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  189. + * See the License for the specific language governing permissions and
  190. + * limitations under the License.
  191. + */
  192. +package org.apache.hadoop.yarn.metrics;
  193. +
  194. +import org.apache.hadoop.classification.InterfaceAudience;
  195. +import org.apache.hadoop.metrics2.MetricsSource;
  196. +import org.apache.hadoop.metrics2.annotation.Metrics;
  197. +
  198. +@InterfaceAudience.Private
  199. +@Metrics(context="yarn")
  200. +public interface EventTypeMetrics<T extends Enum<T>>
  201. + extends MetricsSource {
  202. +
  203. + void increment(T type, long processingTimeUs);
  204. +
  205. + long get(T type);
  206. +}
  207. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/GenericEventTypeMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/GenericEventTypeMetrics.java
  208. new file mode 100644
  209. index 00000000000..464edb27782
  210. --- /dev/null
  211. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/GenericEventTypeMetrics.java
  212. @@ -0,0 +1,160 @@
  213. +/**
  214. + * Licensed to the Apache Software Foundation (ASF) under one
  215. + * or more contributor license agreements. See the NOTICE file
  216. + * distributed with this work for additional information
  217. + * regarding copyright ownership. The ASF licenses this file
  218. + * to you under the Apache License, Version 2.0 (the
  219. + * "License"); you may not use this file except in compliance
  220. + * with the License. You may obtain a copy of the License at
  221. + *
  222. + * http://www.apache.org/licenses/LICENSE-2.0
  223. + *
  224. + * Unless required by applicable law or agreed to in writing, software
  225. + * distributed under the License is distributed on an "AS IS" BASIS,
  226. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  227. + * See the License for the specific language governing permissions and
  228. + * limitations under the License.
  229. + */
  230. +package org.apache.hadoop.yarn.metrics;
  231. +
  232. +import org.apache.hadoop.classification.InterfaceAudience;
  233. +import org.apache.hadoop.metrics2.MetricsCollector;
  234. +import org.apache.hadoop.metrics2.MetricsInfo;
  235. +import org.apache.hadoop.metrics2.MetricsSystem;
  236. +import org.apache.hadoop.metrics2.annotation.Metrics;
  237. +import org.apache.hadoop.metrics2.lib.MetricsRegistry;
  238. +import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
  239. +import org.slf4j.Logger;
  240. +import org.slf4j.LoggerFactory;
  241. +
  242. +import java.util.EnumMap;
  243. +
  244. +@InterfaceAudience.Private
  245. +@Metrics(context="yarn")
  246. +public class GenericEventTypeMetrics<T extends Enum<T>>
  247. + implements EventTypeMetrics<T> {
  248. +
  249. + static final Logger LOG =
  250. + LoggerFactory.getLogger(GenericEventTypeMetrics.class);
  251. +
  252. + private final EnumMap<T, MutableGaugeLong> eventCountMetrics;
  253. + private final EnumMap<T, MutableGaugeLong> processingTimeMetrics;
  254. + private final MetricsRegistry registry;
  255. + private final MetricsSystem ms;
  256. + private final MetricsInfo info;
  257. + private final Class<T> enumClass;
  258. +
  259. + private boolean isInitialized = false;
  260. +
  261. + public GenericEventTypeMetrics(MetricsInfo info, MetricsSystem ms,
  262. + final T[] enums, Class<T> enumClass) {
  263. + this.enumClass = enumClass;
  264. + this.eventCountMetrics = new EnumMap<>(this.enumClass);
  265. + this.processingTimeMetrics = new EnumMap<>(this.enumClass);
  266. + this.ms = ms;
  267. + this.info = info;
  268. + this.registry = new MetricsRegistry(this.info);
  269. +
  270. + //Initialize enum
  271. + for (final T type : enums) {
  272. + String eventCountMetricsName =
  273. + type.toString() + "_" + "event_count";
  274. + String processingTimeMetricsName =
  275. + type.toString() + "_" + "processing_time";
  276. + eventCountMetrics.put(type, this.registry.
  277. + newGauge(eventCountMetricsName, eventCountMetricsName, 0L));
  278. + processingTimeMetrics.put(type, this.registry.
  279. + newGauge(processingTimeMetricsName, processingTimeMetricsName, 0L));
  280. + }
  281. + }
  282. +
  283. + public synchronized GenericEventTypeMetrics registerMetrics() {
  284. + if (!isInitialized) {
  285. + // Register with the MetricsSystems
  286. + if (this.ms != null) {
  287. + LOG.info("Registering GenericEventTypeMetrics");
  288. + ms.register(info.name(),
  289. + info.description(), this);
  290. + isInitialized = true;
  291. + }
  292. + }
  293. + return this;
  294. + }
  295. +
  296. + @Override
  297. + public void increment(T type, long processingTimeUs) {
  298. + if (eventCountMetrics.get(type) != null) {
  299. + eventCountMetrics.get(type).incr();
  300. + processingTimeMetrics.get(type).incr(processingTimeUs);
  301. + }
  302. + }
  303. +
  304. + @Override
  305. + public long get(T type) {
  306. + return eventCountMetrics.get(type).value();
  307. + }
  308. +
  309. + public long getTotalProcessingTime(T type) {
  310. + return processingTimeMetrics.get(type).value();
  311. + }
  312. +
  313. + public EnumMap<T, MutableGaugeLong> getEventCountMetrics() {
  314. + return eventCountMetrics;
  315. + }
  316. +
  317. + public EnumMap<T, MutableGaugeLong> getProcessingTimeMetrics() {
  318. + return processingTimeMetrics;
  319. + }
  320. +
  321. + public MetricsRegistry getRegistry() {
  322. + return registry;
  323. + }
  324. +
  325. + public MetricsInfo getInfo() {
  326. + return info;
  327. + }
  328. +
  329. + @Override
  330. + public void getMetrics(MetricsCollector collector, boolean all) {
  331. + registry.snapshot(collector.addRecord(registry.info()), all);
  332. + }
  333. +
  334. + public Class<T> getEnumClass() {
  335. + return enumClass;
  336. + }
  337. +
  338. + /** Builder class for GenericEventTypeMetrics. */
  339. + public static class EventTypeMetricsBuilder<T extends Enum<T>>{
  340. + public EventTypeMetricsBuilder() {
  341. + }
  342. +
  343. + public EventTypeMetricsBuilder setEnumClass(Class<T> enumClassValue) {
  344. + this.enumClass = enumClassValue;
  345. + return this;
  346. + }
  347. +
  348. + public EventTypeMetricsBuilder setEnums(T[] enumsValue) {
  349. + this.enums = enumsValue.clone();
  350. + return this;
  351. + }
  352. +
  353. + public EventTypeMetricsBuilder setInfo(MetricsInfo infoValue) {
  354. + this.info = infoValue;
  355. + return this;
  356. + }
  357. +
  358. + public EventTypeMetricsBuilder setMs(MetricsSystem msValue) {
  359. + this.ms = msValue;
  360. + return this;
  361. + }
  362. +
  363. + public GenericEventTypeMetrics build() {
  364. + return new GenericEventTypeMetrics(info, ms, enums, enumClass);
  365. + }
  366. +
  367. + private MetricsSystem ms;
  368. + private MetricsInfo info;
  369. + private Class<T> enumClass;
  370. + private T[] enums;
  371. + }
  372. +}
  373. \ No newline at end of file
  374. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
  375. index 55ddd12fce9..7d2572a4c11 100644
  376. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
  377. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
  378. @@ -20,9 +20,20 @@
  379. import java.lang.reflect.Field;
  380. import java.lang.reflect.Modifier;
  381. +import java.util.HashMap;
  382. +import java.util.HashSet;
  383. +import java.util.Map;
  384. +import java.util.Set;
  385. import java.util.concurrent.BlockingQueue;
  386. import java.util.concurrent.LinkedBlockingQueue;
  387. +
  388. +import org.apache.hadoop.metrics2.AbstractMetric;
  389. +import org.apache.hadoop.metrics2.MetricsRecord;
  390. +import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
  391. +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
  392. +import org.apache.hadoop.test.GenericTestUtils;
  393. +import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
  394. import org.slf4j.Logger;
  395. import org.apache.hadoop.conf.Configuration;
  396. import org.apache.hadoop.yarn.conf.YarnConfiguration;
  397. @@ -30,6 +41,7 @@
  398. import org.junit.Assert;
  399. import org.junit.Test;
  400. +import static org.apache.hadoop.metrics2.lib.Interns.info;
  401. import static org.junit.Assert.assertEquals;
  402. import static org.mockito.Mockito.*;
  403. @@ -118,7 +130,7 @@ public void handle(Event event) {
  404. }
  405. private enum TestEnum {
  406. - TestEventType
  407. + TestEventType, TestEventType2
  408. }
  409. @SuppressWarnings({ "rawtypes", "unchecked" })
  410. @@ -230,5 +242,171 @@ public void testPrintDispatcherEventDetailsAvoidDeadLoopInternal()
  411. }
  412. }
  413. -}
  414. + @Test
  415. + public void testMetricsForDispatcher() throws Exception {
  416. + YarnConfiguration conf = new YarnConfiguration();
  417. + AsyncDispatcher dispatcher = null;
  418. +
  419. + try {
  420. + dispatcher = new AsyncDispatcher("RM Event dispatcher");
  421. +
  422. + GenericEventTypeMetrics genericEventTypeMetrics =
  423. + new GenericEventTypeMetrics.EventTypeMetricsBuilder()
  424. + .setMs(DefaultMetricsSystem.instance())
  425. + .setInfo(info("GenericEventTypeMetrics for "
  426. + + TestEnum.class.getName(),
  427. + "Metrics for " + dispatcher.getName()))
  428. + .setEnumClass(TestEnum.class)
  429. + .setEnums(TestEnum.class.getEnumConstants())
  430. + .build().registerMetrics();
  431. +
  432. + // We can the metrics enabled for TestEnum
  433. + dispatcher.addMetrics(genericEventTypeMetrics,
  434. + genericEventTypeMetrics.getEnumClass());
  435. + dispatcher.init(conf);
  436. +
  437. + // Register handler
  438. + dispatcher.register(TestEnum.class, new TestHandler());
  439. + dispatcher.start();
  440. +
  441. + for (int i = 0; i < 3; ++i) {
  442. + Event event = mock(Event.class);
  443. + when(event.getType()).thenReturn(TestEnum.TestEventType);
  444. + dispatcher.getEventHandler().handle(event);
  445. + }
  446. +
  447. + for (int i = 0; i < 2; ++i) {
  448. + Event event = mock(Event.class);
  449. + when(event.getType()).thenReturn(TestEnum.TestEventType2);
  450. + dispatcher.getEventHandler().handle(event);
  451. + }
  452. +
  453. + // Check event type count.
  454. + GenericTestUtils.waitFor(() -> genericEventTypeMetrics.
  455. + get(TestEnum.TestEventType) == 3, 1000, 10000);
  456. +
  457. + GenericTestUtils.waitFor(() -> genericEventTypeMetrics.
  458. + get(TestEnum.TestEventType2) == 2, 1000, 10000);
  459. +
  460. + // Check time spend.
  461. + Assert.assertTrue(genericEventTypeMetrics.
  462. + getTotalProcessingTime(TestEnum.TestEventType)
  463. + >= 1500*3);
  464. + Assert.assertTrue(genericEventTypeMetrics.
  465. + getTotalProcessingTime(TestEnum.TestEventType)
  466. + < 1500*4);
  467. +
  468. + Assert.assertTrue(genericEventTypeMetrics.
  469. + getTotalProcessingTime(TestEnum.TestEventType2)
  470. + >= 1500*2);
  471. + Assert.assertTrue(genericEventTypeMetrics.
  472. + getTotalProcessingTime(TestEnum.TestEventType2)
  473. + < 1500*3);
  474. +
  475. + // Make sure metrics consistent.
  476. + Assert.assertEquals(Long.toString(genericEventTypeMetrics.
  477. + get(TestEnum.TestEventType)),
  478. + genericEventTypeMetrics.
  479. + getRegistry().get("TestEventType_event_count").toString());
  480. + Assert.assertEquals(Long.toString(genericEventTypeMetrics.
  481. + get(TestEnum.TestEventType2)),
  482. + genericEventTypeMetrics.
  483. + getRegistry().get("TestEventType2_event_count").toString());
  484. + Assert.assertEquals(Long.toString(genericEventTypeMetrics.
  485. + getTotalProcessingTime(TestEnum.TestEventType)),
  486. + genericEventTypeMetrics.
  487. + getRegistry().get("TestEventType_processing_time").toString());
  488. + Assert.assertEquals(Long.toString(genericEventTypeMetrics.
  489. + getTotalProcessingTime(TestEnum.TestEventType2)),
  490. + genericEventTypeMetrics.
  491. + getRegistry().get("TestEventType2_processing_time").toString());
  492. +
  493. + } finally {
  494. + dispatcher.close();
  495. + }
  496. +
  497. + }
  498. +
  499. + @Test
  500. + public void testDispatcherMetricsHistogram() throws Exception {
  501. + YarnConfiguration conf = new YarnConfiguration();
  502. + AsyncDispatcher dispatcher = null;
  503. +
  504. + try {
  505. + dispatcher = new AsyncDispatcher("RM Event dispatcher");
  506. +
  507. + GenericEventTypeMetrics genericEventTypeMetrics =
  508. + new GenericEventTypeMetrics.EventTypeMetricsBuilder()
  509. + .setMs(DefaultMetricsSystem.instance())
  510. + .setInfo(info("GenericEventTypeMetrics for "
  511. + + TestEnum.class.getName(),
  512. + "Metrics for " + dispatcher.getName()))
  513. + .setEnumClass(TestEnum.class)
  514. + .setEnums(TestEnum.class.getEnumConstants())
  515. + .build().registerMetrics();
  516. +
  517. + // We can the metrics enabled for TestEnum
  518. + dispatcher.addMetrics(genericEventTypeMetrics,
  519. + genericEventTypeMetrics.getEnumClass());
  520. + dispatcher.init(conf);
  521. +
  522. + // Register handler
  523. + dispatcher.register(TestEnum.class, new TestHandler());
  524. + dispatcher.start();
  525. +
  526. + for (int i = 0; i < 3; ++i) {
  527. + Event event = mock(Event.class);
  528. + when(event.getType()).thenReturn(TestEnum.TestEventType);
  529. + dispatcher.getEventHandler().handle(event);
  530. + }
  531. +
  532. + for (int i = 0; i < 2; ++i) {
  533. + Event event = mock(Event.class);
  534. + when(event.getType()).thenReturn(TestEnum.TestEventType2);
  535. + dispatcher.getEventHandler().handle(event);
  536. + }
  537. +
  538. + // Check event type count.
  539. + GenericTestUtils.waitFor(() -> genericEventTypeMetrics.
  540. + get(TestEnum.TestEventType) == 3, 1000, 10000);
  541. +
  542. + GenericTestUtils.waitFor(() -> genericEventTypeMetrics.
  543. + get(TestEnum.TestEventType2) == 2, 1000, 10000);
  544. +
  545. + // submit actual values
  546. + Map<String, Long> expectedValues = new HashMap<>();
  547. + expectedValues.put("TestEventType_event_count",
  548. + genericEventTypeMetrics.get(TestEnum.TestEventType));
  549. + expectedValues.put("TestEventType_processing_time",
  550. + genericEventTypeMetrics.
  551. + getTotalProcessingTime(TestEnum.TestEventType));
  552. + expectedValues.put("TestEventType2_event_count",
  553. + genericEventTypeMetrics.get(TestEnum.TestEventType2));
  554. + expectedValues.put("TestEventType2_processing_time",
  555. + genericEventTypeMetrics.
  556. + getTotalProcessingTime(TestEnum.TestEventType2));
  557. + Set<String> testResults = new HashSet<>();
  558. + MetricsCollectorImpl collector = new MetricsCollectorImpl();
  559. + genericEventTypeMetrics.getMetrics(collector, true);
  560. +
  561. + for (MetricsRecord record : collector.getRecords()) {
  562. + for (AbstractMetric metric : record.metrics()) {
  563. + String metricName = metric.name();
  564. + if (expectedValues.containsKey(metricName)) {
  565. + Long expectedValue = expectedValues.get(metricName);
  566. + Assert.assertEquals(
  567. + "Metric " + metricName + " doesn't have expected value",
  568. + expectedValue, metric.value());
  569. + testResults.add(metricName);
  570. + }
  571. + }
  572. + }
  573. + Assert.assertEquals(expectedValues.keySet(), testResults);
  574. +
  575. + } finally {
  576. + dispatcher.close();
  577. + }
  578. +
  579. + }
  580. +}
  581. \ No newline at end of file
  582. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/GenericEventTypeMetricsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/GenericEventTypeMetricsManager.java
  583. new file mode 100644
  584. index 00000000000..8fda9b7f38a
  585. --- /dev/null
  586. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/GenericEventTypeMetricsManager.java
  587. @@ -0,0 +1,42 @@
  588. +/**
  589. + * Licensed to the Apache Software Foundation (ASF) under one
  590. + * or more contributor license agreements. See the NOTICE file
  591. + * distributed with this work for additional information
  592. + * regarding copyright ownership. The ASF licenses this file
  593. + * to you under the Apache License, Version 2.0 (the
  594. + * "License"); you may not use this file except in compliance
  595. + * with the License. You may obtain a copy of the License at
  596. + *
  597. + * http://www.apache.org/licenses/LICENSE-2.0
  598. + *
  599. + * Unless required by applicable law or agreed to in writing, software
  600. + * distributed under the License is distributed on an "AS IS" BASIS,
  601. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  602. + * See the License for the specific language governing permissions and
  603. + * limitations under the License.
  604. + */
  605. +package org.apache.hadoop.yarn.server.resourcemanager;
  606. +
  607. +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
  608. +import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
  609. +
  610. +import static org.apache.hadoop.metrics2.lib.Interns.info;
  611. +
  612. +public final class GenericEventTypeMetricsManager {
  613. +
  614. + private GenericEventTypeMetricsManager() {
  615. + // nothing to do
  616. + }
  617. +
  618. + // Construct a GenericEventTypeMetrics for dispatcher
  619. + public static <T extends Enum<T>> GenericEventTypeMetrics
  620. + create(String dispatcherName, Class<T> eventTypeClass) {
  621. + return new GenericEventTypeMetrics.EventTypeMetricsBuilder<T>()
  622. + .setMs(DefaultMetricsSystem.instance())
  623. + .setInfo(info("GenericEventTypeMetrics for " + eventTypeClass.getName(),
  624. + "Metrics for " + dispatcherName))
  625. + .setEnumClass(eventTypeClass)
  626. + .setEnums(eventTypeClass.getEnumConstants())
  627. + .build().registerMetrics();
  628. + }
  629. +}
  630. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  631. index bb8a3ba7db3..ca47fdb648b 100644
  632. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  633. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  634. @@ -21,6 +21,7 @@
  635. import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
  636. import com.sun.jersey.spi.container.servlet.ServletContainer;
  637. +import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
  638. import org.slf4j.Logger;
  639. import org.slf4j.LoggerFactory;
  640. import org.slf4j.Marker;
  641. @@ -63,6 +64,7 @@
  642. import org.apache.hadoop.yarn.event.Dispatcher;
  643. import org.apache.hadoop.yarn.event.EventDispatcher;
  644. import org.apache.hadoop.yarn.event.EventHandler;
  645. +
  646. import org.apache.hadoop.yarn.exceptions.YarnException;
  647. import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
  648. import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
  649. @@ -470,7 +472,14 @@ protected void setRMStateStore(RMStateStore rmStore) {
  650. }
  651. protected Dispatcher createDispatcher() {
  652. - return new AsyncDispatcher("RM Event dispatcher");
  653. + AsyncDispatcher dispatcher = new AsyncDispatcher("RM Event dispatcher");
  654. + GenericEventTypeMetrics genericEventTypeMetrics =
  655. + GenericEventTypeMetricsManager.
  656. + create(dispatcher.getName(), NodesListManagerEventType.class);
  657. + // We can add more
  658. + dispatcher.addMetrics(genericEventTypeMetrics,
  659. + genericEventTypeMetrics.getEnumClass());
  660. + return dispatcher;
  661. }
  662. protected ResourceScheduler createScheduler() {
  663. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
  664. index 1cb5e1d0e76..b9c5500a7d2 100644
  665. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
  666. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
  667. @@ -29,6 +29,7 @@
  668. import org.apache.hadoop.conf.Configuration;
  669. import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
  670. import org.apache.hadoop.http.lib.StaticUserWebFilter;
  671. +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
  672. import org.apache.hadoop.net.NetworkTopology;
  673. import org.apache.hadoop.security.AuthenticationFilterInitializer;
  674. import org.apache.hadoop.security.UserGroupInformation;
  675. @@ -73,6 +74,7 @@
  676. public void setUp() throws Exception {
  677. YarnConfiguration conf = new YarnConfiguration();
  678. UserGroupInformation.setConfiguration(conf);
  679. + DefaultMetricsSystem.setMiniClusterMode(true);
  680. resourceManager = new ResourceManager();
  681. resourceManager.init(conf);
  682. resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  683. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerHealth.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerHealth.java
  684. index a75be7745fb..933eaf91fe4 100644
  685. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerHealth.java
  686. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerHealth.java
  687. @@ -18,6 +18,7 @@
  688. package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
  689. +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
  690. import org.apache.hadoop.net.NetworkTopology;
  691. import org.apache.hadoop.util.Time;
  692. import org.apache.hadoop.yarn.api.records.ContainerId;
  693. @@ -52,6 +53,7 @@
  694. private ResourceManager resourceManager;
  695. public void setup() {
  696. + DefaultMetricsSystem.setMiniClusterMode(true);
  697. resourceManager = new ResourceManager() {
  698. @Override
  699. protected RMNodeLabelsManager createNodeLabelManager() {
  700. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
  701. index 6b0c42f9b16..8bc18481cb2 100644
  702. --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
  703. +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
  704. @@ -210,6 +210,7 @@
  705. @Before
  706. public void setUp() throws Exception {
  707. ResourceUtils.resetResourceTypes(new Configuration());
  708. + DefaultMetricsSystem.setMiniClusterMode(true);
  709. resourceManager = new ResourceManager() {
  710. @Override
  711. protected RMNodeLabelsManager createNodeLabelManager() {