ZOOKEEPER-4930: Add metrics for TTL node deletion (#2261)

Author: Li Wang <liwang@apple.com>

Co-authored-by: liwang <liwang@apple.com>

zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java

@@ -584,6 +584,7 @@ public class DataTree {
                 containers.remove(path);
             } else if (ephemeralType == EphemeralType.TTL) {
                 ttls.remove(path);
+                ServerMetrics.getMetrics().TTL_NODE_DELETED_COUNT.add(1);
             } else if (owner != 0) {
                 Set<String> nodes = ephemerals.get(owner);
                 if (nodes != null) {

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java

@@ -269,6 +269,8 @@ public final class ServerMetrics {
         JVM_PAUSE_TIME = metricsContext.getSummary("jvm_pause_time_ms", DetailLevel.ADVANCED);
 
         QUOTA_EXCEEDED_ERROR_PER_NAMESPACE = metricsContext.getCounterSet(QuotaMetricsUtils.QUOTA_EXCEEDED_ERROR_PER_NAMESPACE);
+
+        TTL_NODE_DELETED_COUNT = metricsContext.getCounter("ttl_node_deleted_count");
     }
 
     /**
@@ -547,6 +549,11 @@ public final class ServerMetrics {
 
     public final CounterSet QUOTA_EXCEEDED_ERROR_PER_NAMESPACE;
 
+    /**
+     * Count of deleted TTL nodes
+     */
+    public final Counter TTL_NODE_DELETED_COUNT;
+
     private final MetricsProvider metricsProvider;
 
     public void resetAll() {

zookeeper-server/src/test/java/org/apache/zookeeper/server/CreateTTLTest.java

@@ -22,11 +22,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.zookeeper.AsyncCallback;
 import org.apache.zookeeper.CreateMode;
@@ -38,6 +40,7 @@ import org.apache.zookeeper.OpResult;
 import org.apache.zookeeper.TestableZooKeeper;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.metrics.MetricsUtils;
 import org.apache.zookeeper.proto.CreateResponse;
 import org.apache.zookeeper.proto.CreateTTLRequest;
 import org.apache.zookeeper.proto.ReplyHeader;
@@ -91,6 +94,10 @@ public class CreateTTLTest extends ClientBase {
         fakeElapsed.set(1000);
         containerManager.checkContainers();
         assertNull(zk.exists("/foo", false), "Ttl node should have been deleted");
+
+        // validate deleted TTL nodes count
+        Map<String, Object> metrics = MetricsUtils.currentServerMetrics();
+        assertTrue((long) metrics.get("ttl_node_deleted_count") >= 1);
     }
 
     @Test