ZOOKEEPER-3327: Add unrecoverable error metric

Author: Jie Huang <jiehuang@fb.com>

Reviewers: andor@apache.org, eolivelli@apache.org, fangmin@apache.org

Closes #862 from jhuan31/ZOOKEEPER-3327

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerMetrics.java

@@ -74,6 +74,8 @@ public enum ServerMetrics {
     CONNECTION_TOKEN_DEFICIT(new AvgMinMaxCounter("connection_token_deficit")),
     CONNECTION_REJECTED(new SimpleCounter("connection_rejected")),
 
+    UNRECOVERABLE_ERROR_COUNT(new SimpleCounter("unrecoverable_error_count")),
+
     BYTES_RECEIVED_COUNT(new SimpleCounter("bytes_received_count")),
 
     /**

zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperCriticalThread.java

@@ -47,5 +47,6 @@ public class ZooKeeperCriticalThread extends ZooKeeperThread {
     protected void handleException(String threadName, Throwable e) {
         LOG.error("Severe unrecoverable error, from thread : {}", threadName, e);
         listener.notifyStopping(threadName, ExitCode.UNEXPECTED_ERROR.getValue());
+        ServerMetrics.UNRECOVERABLE_ERROR_COUNT.add(1);
     }
 }

zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperCriticalThreadMetricsTest.java

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.apache.zookeeper.ZooDefs;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+public class ZooKeeperCriticalThreadMetricsTest extends ZKTestCase {
+    CountDownLatch processed;
+
+    private class MyRequestProcessor implements RequestProcessor {
+        @Override
+        public void processRequest(Request request) throws RequestProcessorException {
+            // use this dummy request processor to trigger a unrecoverable ex
+            throw new RequestProcessorException("test", new Exception());
+        }
+
+        @Override
+        public void shutdown() {
+        }
+    }
+
+    private class MyPrepRequestProcessor extends PrepRequestProcessor {
+        public MyPrepRequestProcessor() {
+            super(new ZooKeeperServer(), new MyRequestProcessor());
+        }
+
+        @Override
+        public void run() {
+            super.run();
+            processed.countDown();
+        }
+
+    }
+
+    @Test
+    public void testUnrecoverableErrorCountFromRequestProcessor() throws Exception{
+        ServerMetrics.resetAll();
+
+        processed = new CountDownLatch(1);
+        PrepRequestProcessor processor =new MyPrepRequestProcessor();
+        processor.start();
+
+        processor.processRequest(new Request(null, 1L, 1, ZooDefs.OpCode.setData,
+                ByteBuffer.wrap(new byte[10]), null));
+        processed.await();
+
+        processor.shutdown();
+
+        Map<String, Object> values = ServerMetrics.getAllValues();
+        Assert.assertEquals(1L, values.get("unrecoverable_error_count"));
+    }
+
+    @Test
+    public void testUnrecoverableErrorCount() {
+        ServerMetrics.resetAll();
+
+        ZooKeeperServer zks = new ZooKeeperServer();
+        ZooKeeperCriticalThread thread = new ZooKeeperCriticalThread("test", zks.getZooKeeperServerListener());
+
+        thread.handleException("test", new Exception());
+
+        Map<String, Object> values = ServerMetrics.getAllValues();
+        Assert.assertEquals(1L, values.get("unrecoverable_error_count"));
+    }
+}