HDFS-3374. TestDelegationToken fails intermittently with a race condition. Contributed by Owen O'Malley.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.0@1334875 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -74,6 +74,9 @@ Release 1.0.3 - unreleased
     HADOOP-8151. Error handling in snappy decompressor throws invalid
     exceptions. (Matt Foley)
 
+    HDFS-3374. hdfs' TestDelegationToken fails intermittently with a race
+    condition. (Owen O'Malley via mattf)
+
 Release 1.0.2 - 2012.03.24
 
   NEW FEATURES

src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java

@@ -381,24 +381,30 @@ extends AbstractDelegationTokenIdentifier>
         while (running) {
           long now = System.currentTimeMillis();
           if (lastMasterKeyUpdate + keyUpdateInterval < now) {
-            try {
-              rollMasterKey();
-              lastMasterKeyUpdate = now;
-            } catch (IOException e) {
-              LOG.error("Master key updating failed. "
-                  + StringUtils.stringifyException(e));
+            synchronized (AbstractDelegationTokenSecretManager.this) {
+              if (running) {
+                try {
+                  rollMasterKey();
+                  lastMasterKeyUpdate = now;
+                } catch (IOException e) {
+                  LOG.error("Master key updating failed. "
+                            + StringUtils.stringifyException(e));
+                }
+              }
             }
           }
           if (lastTokenCacheCleanup + tokenRemoverScanInterval < now) {
-            removeExpiredToken();
-            lastTokenCacheCleanup = now;
+            synchronized (AbstractDelegationTokenSecretManager.this) {
+              if (running) {
+                removeExpiredToken();
+                lastTokenCacheCleanup = now;
+              }
+            }
           }
           try {
             Thread.sleep(5000); // 5 seconds
           } catch (InterruptedException ie) {
-            LOG
-            .error("InterruptedExcpetion recieved for ExpiredTokenRemover thread "
-                + ie);
+            return;
           }
         }
       } catch (Throwable t) {

src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java

@@ -69,13 +69,18 @@ public class TestDelegationToken {
     cluster.waitActive();
     cluster.getNameNode().getNamesystem().getDelegationTokenSecretManager()
 				.startThreads();
+    LOG.info("cluster up and running");
   }
 
   @After
   public void tearDown() throws Exception {
+    LOG.info("starting shutting down the cluster");
     if(cluster!=null) {
+      cluster.getNameNode().getNamesystem().getDelegationTokenSecretManager()
+             .stopThreads();
       cluster.shutdown();
     }
+    LOG.info("finished shutting down the cluster");
   }
 
   private Token<DelegationTokenIdentifier> generateDelegationToken(