ZOOKEEPER-3251: Add new server metric types: percentile counter and c…

…ounter set

Author: Jie Huang <jiehuang@fb.com>

Reviewers: fangmin@apache.org, andor@apache.org

Closes #781 from jhuan31/ZOOKEEPER-3251 and squashes the following commits:

c3f4f7f40 [Jie Huang] Make new unit test classes extend ZKTestCase
50a631f54 [Jie Huang] Add Apache license header
e9b7c9500 [Jie Huang] ZOOKEEPER-3251: Add new server metric types: percentile counter and counter set

ivy.xml

@@ -136,7 +136,8 @@
 
     <dependency org="org.eclipse.jetty" name="jetty-server" rev="${jetty.version}"
                 conf="optional->default"/>
-      <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="${jetty.version}"
+
+    <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="${jetty.version}"
                   conf="optional->default"/>
     <dependency org="com.fasterxml.jackson.core" name="jackson-databind"
                 rev="${jackson.version}" conf="optional->default"/>
@@ -149,7 +150,11 @@
     <dependency org="org.openjdk.jmh" name="jmh-core" rev="${jmh.version}" conf="test->default"/>
     <dependency org="org.openjdk.jmh" name="jmh-generator-annprocess" rev="${jmh.version}" conf="test->default"/>
 
-    <conflict manager="strict"/>
+    <dependency org="io.dropwizard.metrics" name="metrics-core"
+                rev="3.2.5" conf="default">
+      <exclude org="org.slf4j" module="slf4j-api"/>
+    </dependency>
 
+    <conflict manager="strict"/>
   </dependencies>
 </ivy-module>

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java

@@ -19,6 +19,9 @@
 package org.apache.zookeeper.server;
 
 import org.apache.zookeeper.server.metric.AvgMinMaxCounter;
+import org.apache.zookeeper.server.metric.AvgMinMaxCounterSet;
+import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounter;
+import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounterSet;
 import org.apache.zookeeper.server.metric.Metric;
 import org.apache.zookeeper.server.metric.SimpleCounter;
 
@@ -45,20 +48,20 @@ public enum ServerMetrics {
      * Stats for read request. The timing start from when the server see the
      * request until it leave final request processor.
      */
-    READ_LATENCY(new AvgMinMaxCounter("readlatency")),
+    READ_LATENCY(new AvgMinMaxPercentileCounter("readlatency")),
 
     /**
      * Stats for request that need quorum voting. Timing is the same as read
      * request. We only keep track of stats for request that originated from
      * this machine only.
      */
-    UPDATE_LATENCY(new AvgMinMaxCounter("updatelatency")),
+    UPDATE_LATENCY(new AvgMinMaxPercentileCounter("updatelatency")),
 
     /**
      * Stats for all quorum request. The timing start from when the leader
      * see the request until it reach the learner.
      */
-    PROPAGATION_LATENCY(new AvgMinMaxCounter("propagation_latency")),
+    PROPAGATION_LATENCY(new AvgMinMaxPercentileCounter("propagation_latency")),
 
     FOLLOWER_SYNC_TIME(new AvgMinMaxCounter("follower_sync_time")),
     ELECTION_TIME(new AvgMinMaxCounter("election_time")),
@@ -86,6 +89,14 @@ public enum ServerMetrics {
         metric.add(value);
     }
 
+    public void add(int key, long value) {
+        metric.add(key, value);
+    }
+
+    public void add(String key, long value) {
+        metric.add(key, value);
+    }
+
     public void reset() {
         metric.reset();
     }

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/AvgMinMaxCounter.java

@@ -27,7 +27,7 @@ import java.util.concurrent.atomic.AtomicLong;
  * Generic long counter that keep track of min/max/avg. The counter is
  * thread-safe
  */
-public class AvgMinMaxCounter implements Metric {
+public class AvgMinMaxCounter extends Metric {
     private String name;
     private AtomicLong total = new AtomicLong();
     private AtomicLong min = new AtomicLong(Long.MAX_VALUE);
@@ -111,6 +111,8 @@ public class AvgMinMaxCounter implements Metric {
         m.put("min_" + name, this.getMin());
         m.put("max_" + name, this.getMax());
         m.put("cnt_" + name, this.getCount());
+        m.put("sum_" + name, this.getTotal());
         return m;
     }
+
 }

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/AvgMinMaxCounterSet.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.metric;
+
+import java.lang.Integer;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generic set of long counters that keep track of min/max/avg
+ * for different keys.
+ * The counter is thread-safe
+ */
+public class AvgMinMaxCounterSet extends Metric {
+    private String name;
+
+    private ConcurrentHashMap<String, AvgMinMaxCounter> counters = new ConcurrentHashMap<>();
+
+    public AvgMinMaxCounterSet(String name) {
+        this.name = name;
+    }
+
+    private AvgMinMaxCounter getCounterForKey(String key) {
+        AvgMinMaxCounter counter = counters.get(key);
+        if (counter == null) {
+            counters.putIfAbsent(key, new AvgMinMaxCounter(key + "_" + name));
+            counter = counters.get(key);
+        }
+
+        return counter;
+    }
+
+    public void addDataPoint(String key, long value) {
+        getCounterForKey(key).addDataPoint(value);
+    }
+
+    public void resetMax() {
+        for (Map.Entry<String, AvgMinMaxCounter> entry : counters.entrySet()) {
+            entry.getValue().resetMax();
+        }
+    }
+
+    public void reset() {
+        for (Map.Entry<String, AvgMinMaxCounter> entry : counters.entrySet()) {
+            entry.getValue().reset();
+        }
+    }
+
+    @Override
+    public void add(String key, long value) {
+        addDataPoint(key, value);
+    }
+
+    @Override
+    public Map<String, Object> values() {
+        Map<String, Object> m = new LinkedHashMap<>();
+        for (Map.Entry<String, AvgMinMaxCounter> entry : counters.entrySet()) {
+            m.putAll(entry.getValue().values());
+        }
+        return m;
+    }
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/AvgMinMaxPercentileCounter.java

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server.metric;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Reservoir;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.UniformSnapshot;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+
+/**
+ * Generic long counter that keep track of min/max/avg/percentiles.
+ * The counter is thread-safe
+ */
+public class AvgMinMaxPercentileCounter extends Metric  {
+
+    private String name;
+    private AvgMinMaxCounter counter;
+    private final ResettableUniformReservoir reservoir;
+    private final Histogram histogram;
+
+    static class ResettableUniformReservoir implements Reservoir {
+
+        private static final int DEFAULT_SIZE = 4096;
+        private static final int BITS_PER_LONG = 63;
+
+        private final AtomicLong count = new AtomicLong();
+        private volatile AtomicLongArray values = new AtomicLongArray(DEFAULT_SIZE);
+
+        @Override
+        public int size() {
+            final long c = count.get();
+            if (c > values.length()) {
+                return values.length();
+            }
+            return (int) c;
+        }
+
+        @Override
+        public void update(long value) {
+            final long c = count.incrementAndGet();
+            if (c <= values.length()) {
+                values.set((int) c - 1, value);
+            } else {
+                final long r = nextLong(c);
+                if (r < values.length()) {
+                    values.set((int) r, value);
+                }
+            }
+        }
+
+        private static long nextLong(long n) {
+            long bits, val;
+            do {
+                bits = ThreadLocalRandom.current().nextLong() & (~(1L << BITS_PER_LONG));
+                val = bits % n;
+            } while (bits - val + (n - 1) < 0L);
+            return val;
+        }
+
+        @Override
+        public Snapshot getSnapshot() {
+            final int s = size();
+            final List<Long> copy = new ArrayList<Long>(s);
+            for (int i = 0; i < s; i++) {
+                copy.add(values.get(i));
+            }
+            return new UniformSnapshot(copy);
+        }
+
+        public void reset() {
+            count.set(0);
+            values = new AtomicLongArray(DEFAULT_SIZE);
+        }
+    }
+
+    public AvgMinMaxPercentileCounter(String name) {
+
+        this.name = name;
+        this.counter = new AvgMinMaxCounter(this.name);
+        reservoir = new ResettableUniformReservoir();
+        histogram = new Histogram(reservoir);
+    }
+
+    public void addDataPoint(long value) {
+        counter.add(value);
+        histogram.update(value);
+    }
+
+    public void resetMax() {
+        // To match existing behavior in upstream
+        counter.resetMax();
+    }
+
+    public void reset() {
+        counter.reset();
+        reservoir.reset();
+    }
+
+    public void add(long value) {
+        addDataPoint(value);
+    }
+
+    public Map<String, Object> values() {
+        Map<String, Object> m = new LinkedHashMap<>();
+        m.putAll(counter.values());
+        m.put("p50_" + name, Math.round(this.histogram.getSnapshot().getMedian()));
+        m.put("p95_" + name, Math.round(this.histogram.getSnapshot().get95thPercentile()));
+        m.put("p99_" + name, Math.round(this.histogram.getSnapshot().get99thPercentile()));
+        m.put("p999_" + name, Math.round(this.histogram.getSnapshot().get999thPercentile()));
+        return m;
+    }
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/AvgMinMaxPercentileCounterSet.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.metric;
+
+import java.lang.Integer;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generic set of long counters that keep track of min/max/avg
+ * for different keys.
+ * The counter is thread-safe
+ */
+public class AvgMinMaxPercentileCounterSet extends Metric {
+    private String name;
+
+    private ConcurrentHashMap<String, AvgMinMaxPercentileCounter> counters = new ConcurrentHashMap<>();
+
+    public AvgMinMaxPercentileCounterSet(String name) {
+        this.name = name;
+    }
+
+    private AvgMinMaxPercentileCounter getCounterForKey(String key) {
+        AvgMinMaxPercentileCounter counter = counters.get(key);
+        if (counter == null) {
+            counters.putIfAbsent(key, new AvgMinMaxPercentileCounter(key + "_" + name));
+            counter = counters.get(key);
+        }
+
+        return counter;
+    }
+
+    public void addDataPoint(String key, long value) {
+        getCounterForKey(key).addDataPoint(value);
+    }
+
+    public void resetMax() {
+        for (Map.Entry<String, AvgMinMaxPercentileCounter> entry : counters.entrySet()) {
+            entry.getValue().resetMax();
+        }
+    }
+
+    public void reset() {
+        for (Map.Entry<String, AvgMinMaxPercentileCounter> entry : counters.entrySet()) {
+            entry.getValue().reset();
+        }
+    }
+
+    @Override
+    public void add(String key, long value) {
+        addDataPoint(key, value);
+    }
+
+    @Override
+    public Map<String, Object> values() {
+        Map<String, Object> m = new LinkedHashMap<>();
+        for (Map.Entry<String, AvgMinMaxPercentileCounter> entry : counters.entrySet()) {
+            m.putAll(entry.getValue().values());
+        }
+        return m;
+    }
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/Metric.java

@@ -20,8 +20,10 @@ package org.apache.zookeeper.server.metric;
 
 import java.util.Map;
 
-public interface Metric {
-    void add(long value);
-    void reset();
-    Map<String, Object> values();
+public abstract class Metric {
+    public void add(long value) {};
+    public void add(int key, long value) {};
+    public void add(String key, long value) {};
+    public void reset() {};
+    public abstract Map<String, Object > values();
 }

zookeeper-server/src/main/java/org/apache/zookeeper/server/metric/SimpleCounter.java

@@ -18,11 +18,12 @@
 
 package org.apache.zookeeper.server.metric;
 
+import java.lang.Override;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
-public class SimpleCounter implements Metric {
+public class SimpleCounter extends Metric {
     private final String name;
     private final AtomicLong counter = new AtomicLong();
 

zookeeper-server/src/test/java/org/apache/zookeeper/server/metric/AvgMinMaxCounterSetTest.java

@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server.metric;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class AvgMinMaxCounterSetTest extends ZKTestCase {
+    private AvgMinMaxCounterSet testCounterSet;
+
+    @Before
+    public void initCounter() {
+        testCounterSet = new AvgMinMaxCounterSet("test");
+    }
+
+    private void addDataPoints() {
+        testCounterSet.add("key1", 0);
+        testCounterSet.add("key1", 1);
+        testCounterSet.add("key2", 2);
+        testCounterSet.add("key2", 3);
+        testCounterSet.add("key2", 4);
+        testCounterSet.add("key2", 5);
+    }
+
+    @Test
+    public void testReset() {
+        addDataPoints();
+        testCounterSet.reset();
+
+        Map<String, Object> values = testCounterSet.values();
+
+        Assert.assertEquals("There should be 10 values in the set", 10, values.size());
+
+        Assert.assertEquals("avg_key1_test should =0", 0D, values.get("avg_key1_test"));
+        Assert.assertEquals("min_key1_test should =0", 0L, values.get("min_key1_test"));
+        Assert.assertEquals("max_key1_test should =0", 0L, values.get("max_key1_test"));
+        Assert.assertEquals("cnt_key1_test should =0", 0L, values.get("cnt_key1_test"));
+        Assert.assertEquals("sum_key1_test should =0", 0L, values.get("sum_key1_test"));
+
+        Assert.assertEquals("avg_key2_test should =0", 0D, values.get("avg_key2_test"));
+        Assert.assertEquals("min_key2_test should =0", 0L, values.get("min_key2_test"));
+        Assert.assertEquals("max_key2_test should =0", 0L, values.get("max_key2_test"));
+        Assert.assertEquals("cnt_key2_test should =0", 0L, values.get("cnt_key2_test"));
+        Assert.assertEquals("sum_key2_test should =0", 0L, values.get("sum_key2_test"));
+
+    }
+
+    @Test
+    public void testValues() {
+        addDataPoints();
+        Map<String, Object> values = testCounterSet.values();
+
+        Assert.assertEquals("There should be 10 values in the set", 10, values.size());
+        Assert.assertEquals("avg_key1_test should =0.5", 0.5D, values.get("avg_key1_test"));
+        Assert.assertEquals("min_key1_test should =0", 0L, values.get("min_key1_test"));
+        Assert.assertEquals("max_key1_test should =1", 1L, values.get("max_key1_test"));
+        Assert.assertEquals("cnt_key1_test should =2", 2L, values.get("cnt_key1_test"));
+        Assert.assertEquals("sum_key1_test should =1", 1L, values.get("sum_key1_test"));
+
+        Assert.assertEquals("avg_key2_test should =3.5", 3.5, values.get("avg_key2_test"));
+        Assert.assertEquals("min_key2_test should =2", 2L, values.get("min_key2_test"));
+        Assert.assertEquals("max_key2_test should =5", 5L, values.get("max_key2_test"));
+        Assert.assertEquals("cnt_key2_test should =4", 4L, values.get("cnt_key2_test"));
+        Assert.assertEquals("sum_key2_test should =14", 14L, values.get("sum_key2_test"));
+    }
+}

zookeeper-server/src/test/java/org/apache/zookeeper/server/metric/AvgMinMaxPercentileCounterSetTest.java

@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server.metric;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class AvgMinMaxPercentileCounterSetTest extends ZKTestCase {
+    private AvgMinMaxPercentileCounterSet testCounterSet;
+
+    @Before
+    public void initCounter() {
+        testCounterSet = new AvgMinMaxPercentileCounterSet("test");
+    }
+
+    private void addDataPoints() {
+        for (int i=0; i<1000; i++) {
+            testCounterSet.add("key1", i);
+        }
+
+        for (int i=1000; i<2000; i++) {
+            testCounterSet.add("key2", i);
+        }
+    }
+
+    @Test
+    public void testReset() {
+        addDataPoints();
+        testCounterSet.reset();
+
+        Map<String, Object> values = testCounterSet.values();
+
+        Assert.assertEquals("avg_key1_test should =0", 0D, values.get("avg_key1_test"));
+        Assert.assertEquals("min_key1_test should =0", 0L, values.get("min_key1_test"));
+        Assert.assertEquals("max_key1_test should =0", 0L, values.get("max_key1_test"));
+        Assert.assertEquals("cnt_key1_test should =0", 0L, values.get("cnt_key1_test"));
+        Assert.assertEquals("sum_key1_test should =0", 0L, values.get("sum_key1_test"));
+        Assert.assertEquals("p50_key1_test should have p50=0", 0L, values.get("p50_key1_test"));
+        Assert.assertEquals("p95_key1_test should have p95=0", 0L, values.get("p95_key1_test"));
+        Assert.assertEquals("p99_key1_test should have p99=0", 0L, values.get("p99_key1_test"));
+        Assert.assertEquals("p999_key1_test should have p999=0", 0L, values.get("p999_key1_test"));
+
+        Assert.assertEquals("avg_key2_test should =0", 0D, values.get("avg_key2_test"));
+        Assert.assertEquals("min_key2_test should =0", 0L, values.get("min_key2_test"));
+        Assert.assertEquals("max_key2_test should =0", 0L, values.get("max_key2_test"));
+        Assert.assertEquals("cnt_key2_test should =0", 0L, values.get("cnt_key2_test"));
+        Assert.assertEquals("sum_key2_test should =0", 0L, values.get("sum_key2_test"));
+        Assert.assertEquals("p50_key2_test should have p50=0", 0L, values.get("p50_key2_test"));
+        Assert.assertEquals("p95_key2_test should have p95=0", 0L, values.get("p95_key2_test"));
+        Assert.assertEquals("p99_key2_test should have p99=0", 0L, values.get("p99_key2_test"));
+        Assert.assertEquals("p999_key2_test should have p999=0", 0L, values.get("p999_key2_test"));
+    }
+
+    @Test
+    public void testValues() {
+        addDataPoints();
+        Map<String, Object> values = testCounterSet.values();
+
+        Assert.assertEquals("There should be 18 values in the set", 18, values.size());
+
+        Assert.assertEquals("avg_key1_test should =499.5", 999D/2, values.get("avg_key1_test"));
+        Assert.assertEquals("min_key1_test should =0", 0L, values.get("min_key1_test"));
+        Assert.assertEquals("max_key1_test should =999", 999L, values.get("max_key1_test"));
+        Assert.assertEquals("cnt_key1_test should =1000", 1000L, values.get("cnt_key1_test"));
+        Assert.assertEquals("sum_key1_test should =999*500", 999*500L, values.get("sum_key1_test"));
+        Assert.assertEquals("p50_key1_test should have p50=500", 500L, values.get("p50_key1_test"));
+        Assert.assertEquals("p95_key1_test should have p95=950", 950L, values.get("p95_key1_test"));
+        Assert.assertEquals("p99_key1_test should have p99=990", 990L, values.get("p99_key1_test"));
+        Assert.assertEquals("p999_key1_test should have p999=999", 999L, values.get("p999_key1_test"));
+
+        Assert.assertEquals("avg_key2_test should =3.5", 1000+999D/2, values.get("avg_key2_test"));
+        Assert.assertEquals("min_key2_test should =2", 1000L, values.get("min_key2_test"));
+        Assert.assertEquals("max_key2_test should =5", 1999L, values.get("max_key2_test"));
+        Assert.assertEquals("cnt_key2_test should =4", 1000L, values.get("cnt_key2_test"));
+        Assert.assertEquals("sum_key2_test should =14", 2999*500L, values.get("sum_key2_test"));
+        Assert.assertEquals("p50_key2_test should have p50=1500", 1500L, values.get("p50_key2_test"));
+        Assert.assertEquals("p95_key2_test should have p95=1950", 1950L, values.get("p95_key2_test"));
+        Assert.assertEquals("p99_key2_test should have p99=1990", 1990L, values.get("p99_key2_test"));
+        Assert.assertEquals("p999_key2_test should have p999=1999", 1999L, values.get("p999_key2_test"));
+    }
+}

zookeeper-server/src/test/java/org/apache/zookeeper/server/metric/AvgMinMaxPercentileCounterTest.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server.metric;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class AvgMinMaxPercentileCounterTest extends ZKTestCase {
+
+    private AvgMinMaxPercentileCounter testCounter;
+
+    @Before
+    public void initCounter() {
+        testCounter = new AvgMinMaxPercentileCounter("test");
+    }
+
+    private void addDataPoints() {
+        for (int i=0; i<1000; i++) {
+            testCounter.add(i);
+        }
+    }
+
+
+    @Test
+    public void testReset() {
+        addDataPoints();
+        testCounter.reset();
+
+        Map<String, Object> values = testCounter.values();
+
+        Assert.assertEquals("There should be 9 values in the set", 9, values.size());
+
+        Assert.assertEquals("should avg=0", 0D, values.get("avg_test"));
+        Assert.assertEquals("should have min=0", 0L, values.get("min_test"));
+        Assert.assertEquals("should have max=0", 0L, values.get("max_test"));
+        Assert.assertEquals("should have cnt=0", 0L, values.get("cnt_test"));
+        Assert.assertEquals("should have sum=0", 0L, values.get("sum_test"));
+        Assert.assertEquals("should have p50=0", 0L, values.get("p50_test"));
+        Assert.assertEquals("should have p95=0", 0L, values.get("p95_test"));
+        Assert.assertEquals("should have p99=0", 0L, values.get("p99_test"));
+        Assert.assertEquals("should have p999=0", 0L, values.get("p999_test"));
+    }
+
+    @Test
+    public void testValues() {
+        addDataPoints();
+        Map<String, Object> values = testCounter.values();
+
+        Assert.assertEquals("There should be 9 values in the set", 9, values.size());
+
+        Assert.assertEquals("should avg=499.5", 999D/2, values.get("avg_test"));
+        Assert.assertEquals("should have min=0", 0L, values.get("min_test"));
+        Assert.assertEquals("should have max=999", 999L, values.get("max_test"));
+        Assert.assertEquals("should have cnt=1000", 1000L, values.get("cnt_test"));
+        Assert.assertEquals("should have sum=999*500", 999*500L, values.get("sum_test"));
+        Assert.assertEquals("should have p50=500", 500L, values.get("p50_test"));
+        Assert.assertEquals("should have p95=950", 950L, values.get("p95_test"));
+        Assert.assertEquals("should have p99=990", 990L, values.get("p99_test"));
+        Assert.assertEquals("should have p999=999", 999L, values.get("p999_test"));
+    }
+}