HADOOP-16278. With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die.

Contributed by Rajat Khandelwal

(cherry picked from commit 591ca698230f25217c10c7549aff8097baa11f1e)

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java

@@ -120,7 +120,8 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
   private final MutableCounterLong streamBytesReadInClose;
   private final MutableCounterLong streamBytesDiscardedInAbort;
   private final MutableCounterLong ignoredErrors;
-
+  private final MutableQuantiles putLatencyQuantile;
+  private final MutableQuantiles throttleRateQuantile;
   private final MutableCounterLong numberOfFilesCreated;
   private final MutableCounterLong numberOfFilesCopied;
   private final MutableCounterLong bytesOfFilesCopied;
@@ -236,9 +237,9 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
     }
     //todo need a config for the quantiles interval?
     int interval = 1;
-    quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
+    putLatencyQuantile = quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
         "ops", "latency", interval);
-    quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
+    throttleRateQuantile = quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
         "events", "frequency (Hz)", interval);
 
     registerAsMetricsSource(name);
@@ -603,6 +604,8 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
 
   public void close() {
     synchronized (metricsSystemLock) {
+      putLatencyQuantile.stop();
+      throttleRateQuantile.stop();
       metricsSystem.unregisterSource(metricsSourceName);
       int activeSources = --metricsSourceActiveCounter;
       if (activeSources == 0) {