HADOOP-11722. Some Instances of Services using ZKDelegationTokenSecretManager go down when old token cannot be deleted. Contributed by Arun Suresh.

(cherry picked from commit fc90bf7b27cc20486f2806670a14fd7d654b0a31)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -700,6 +700,10 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11720. [JDK8] Fix javadoc errors caused by incorrect or illegal
     tags in hadoop-tools. (Akira AJISAKA via ozawa)
 
+    HADOOP-11722. Some Instances of Services using
+    ZKDelegationTokenSecretManager go down when old token cannot be deleted.
+    (Arun Suresh via atm)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java

@@ -55,6 +55,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.ZooDefs.Perms;
 import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.data.ACL;
@@ -709,7 +710,15 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
     try {
       if (zkClient.checkExists().forPath(nodeRemovePath) != null) {
         while(zkClient.checkExists().forPath(nodeRemovePath) != null){
-          zkClient.delete().guaranteed().forPath(nodeRemovePath);
+          try {
+            zkClient.delete().guaranteed().forPath(nodeRemovePath);
+          } catch (NoNodeException nne) {
+            // It is possible that the node might be deleted between the
+            // check and the actual delete.. which might lead to an
+            // exception that can bring down the daemon running this
+            // SecretManager
+            LOG.debug("Node already deleted by peer " + nodeRemovePath);
+          }
         }
       } else {
         LOG.debug("Attempted to delete a non-existing znode " + nodeRemovePath);
@@ -761,7 +770,15 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
     try {
       if (zkClient.checkExists().forPath(nodeRemovePath) != null) {
         while(zkClient.checkExists().forPath(nodeRemovePath) != null){
-          zkClient.delete().guaranteed().forPath(nodeRemovePath);
+          try {
+            zkClient.delete().guaranteed().forPath(nodeRemovePath);
+          } catch (NoNodeException nne) {
+            // It is possible that the node might be deleted between the
+            // check and the actual delete.. which might lead to an
+            // exception that can bring down the daemon running this
+            // SecretManager
+            LOG.debug("Node already deleted by peer " + nodeRemovePath);
+          }
         }
       } else {
         LOG.debug("Attempted to remove a non-existing znode " + nodeRemovePath);