ZOOKEEPER-3859: Add a couple request processor metrics

Author: Jie Huang <jiehuang@fb.com>

Reviewers: Enrico Olivelli <eolivelli@apache.org>, Michael Han <hanm@apache.org>

Closes #1379 from jhuan31/ZOOKEEPER-3859

zookeeper-server/src/main/java/org/apache/zookeeper/server/PrepRequestProcessor.java

@@ -771,8 +771,10 @@ public class PrepRequestProcessor extends ZooKeeperCriticalThread implements Req
         }
 
         request.zxid = zks.getZxid();
-        ServerMetrics.getMetrics().PREP_PROCESS_TIME.add(Time.currentElapsedTime() - request.prepStartTime);
+        long timeFinishedPrepare = Time.currentElapsedTime();
+        ServerMetrics.getMetrics().PREP_PROCESS_TIME.add(timeFinishedPrepare - request.prepStartTime);
         nextProcessor.processRequest(request);
+        ServerMetrics.getMetrics().PROPOSAL_PROCESS_TIME.add(Time.currentElapsedTime() - timeFinishedPrepare);
     }
 
     /**

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java

@@ -123,6 +123,7 @@ public final class ServerMetrics {
         OUTSTANDING_CHANGES_QUEUED = metricsContext.getCounter("outstanding_changes_queued");
         OUTSTANDING_CHANGES_REMOVED = metricsContext.getCounter("outstanding_changes_removed");
         PREP_PROCESS_TIME = metricsContext.getSummary("prep_process_time", DetailLevel.BASIC);
+        PROPOSAL_PROCESS_TIME = metricsContext.getSummary("proposal_process_time", DetailLevel.BASIC);
         CLOSE_SESSION_PREP_TIME = metricsContext.getSummary("close_session_prep_time", DetailLevel.ADVANCED);
 
         REVALIDATE_COUNT = metricsContext.getCounter("revalidate_count");
@@ -236,6 +237,8 @@ public final class ServerMetrics {
 
         DIGEST_MISMATCHES_COUNT = metricsContext.getCounter("digest_mismatches_count");
 
+        LEARNER_REQUEST_PROCESSOR_QUEUE_SIZE = metricsContext.getSummary("learner_request_processor_queue_size", DetailLevel.BASIC);
+
         UNSUCCESSFUL_HANDSHAKE = metricsContext.getCounter("unsuccessful_handshake");
         INSECURE_ADMIN = metricsContext.getCounter("insecure_admin_count");
         TLS_HANDSHAKE_EXCEEDED = metricsContext.getCounter("tls_handshake_exceeded");
@@ -325,6 +328,7 @@ public final class ServerMetrics {
     public final Counter OUTSTANDING_CHANGES_QUEUED;
     public final Counter OUTSTANDING_CHANGES_REMOVED;
     public final Summary PREP_PROCESS_TIME;
+    public final Summary PROPOSAL_PROCESS_TIME;
     public final Summary CLOSE_SESSION_PREP_TIME;
 
     public final Summary PROPOSAL_LATENCY;
@@ -472,6 +476,8 @@ public final class ServerMetrics {
     // txns to data tree.
     public final Counter DIGEST_MISMATCHES_COUNT;
 
+    public final Summary LEARNER_REQUEST_PROCESSOR_QUEUE_SIZE;
+
     public final Counter UNSUCCESSFUL_HANDSHAKE;
 
     /*

zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerRequestProcessor.java

@@ -64,6 +64,8 @@ public class FollowerRequestProcessor extends ZooKeeperCriticalThread implements
     public void run() {
         try {
             while (!finished) {
+                ServerMetrics.getMetrics().LEARNER_REQUEST_PROCESSOR_QUEUE_SIZE.add(queuedRequests.size());
+
                 Request request = queuedRequests.take();
                 if (LOG.isTraceEnabled()) {
                     ZooTrace.logRequest(LOG, ZooTrace.CLIENT_REQUEST_TRACE_MASK, 'F', request, "");
@@ -118,6 +120,8 @@ public class FollowerRequestProcessor extends ZooKeeperCriticalThread implements
                     break;
                 }
             }
+        } catch (RuntimeException e) { // spotbugs require explicit catch of RuntimeException
+            handleException(this.getName(), e);
         } catch (Exception e) {
             handleException(this.getName(), e);
         }

zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverRequestProcessor.java

@@ -24,6 +24,7 @@ import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.OpCode;
 import org.apache.zookeeper.server.Request;
 import org.apache.zookeeper.server.RequestProcessor;
+import org.apache.zookeeper.server.ServerMetrics;
 import org.apache.zookeeper.server.ZooKeeperCriticalThread;
 import org.apache.zookeeper.server.ZooTrace;
 import org.apache.zookeeper.txn.ErrorTxn;
@@ -64,6 +65,8 @@ public class ObserverRequestProcessor extends ZooKeeperCriticalThread implements
     public void run() {
         try {
             while (!finished) {
+                ServerMetrics.getMetrics().LEARNER_REQUEST_PROCESSOR_QUEUE_SIZE.add(queuedRequests.size());
+
                 Request request = queuedRequests.take();
                 if (LOG.isTraceEnabled()) {
                     ZooTrace.logRequest(LOG, ZooTrace.CLIENT_REQUEST_TRACE_MASK, 'F', request, "");
@@ -118,6 +121,8 @@ public class ObserverRequestProcessor extends ZooKeeperCriticalThread implements
                     break;
                 }
             }
+        } catch (RuntimeException e) { // spotbugs require explicit catch of RuntimeException
+            handleException(this.getName(), e);
         } catch (Exception e) {
             handleException(this.getName(), e);
         }