MAPREDUCE-4861. Cleanup: Remove unused mapreduce.security.token.DelegationTokenRenewal. (kkambatl via tucu)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1420350 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -54,6 +54,9 @@ Release 2.0.3-alpha - Unreleased
     MAPREDUCE-4800. Cleanup o.a.h.mapred.MapTaskStatus - remove unused 
     code. (kkambatl via tucu)
 
+    MAPREDUCE-4861. Cleanup: Remove unused mapreduce.security.token.DelegationTokenRenewal. 
+    (kkambatl via tucu)
+
 Release 2.0.2-alpha - 2012-09-07 
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/dev-support/findbugs-exclude.xml

@@ -138,11 +138,6 @@
        <Method name="run" />
        <Bug pattern="DM_EXIT" />
      </Match>
-     <Match>
-       <Class name="org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal$DelegationTokenCancelThread" />
-       <Method name="run" />
-       <Bug pattern="DM_EXIT" />
-    </Match>
      <!--
        We need to cast objects between old and new api objects
      -->

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java

@@ -1,318 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.security.token;
-
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.StringUtils;
-
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class DelegationTokenRenewal {
-  private static final Log LOG = LogFactory.getLog(DelegationTokenRenewal.class);
-  public static final String SCHEME = "hdfs";
-  
-  /**
-   * class that is used for keeping tracks of DT to renew
-   *
-   */
-  private static class DelegationTokenToRenew {
-    public final Token<?> token;
-    public final JobID jobId;
-    public final Configuration conf;
-    public long expirationDate;
-    public TimerTask timerTask;
-    
-    public DelegationTokenToRenew(
-        JobID jId, Token<?> t, 
-        Configuration newConf, long newExpirationDate) {
-      token = t;
-      jobId = jId;
-      conf = newConf;
-      expirationDate = newExpirationDate;
-      timerTask = null;
-      if(token==null || jobId==null || conf==null) {
-        throw new IllegalArgumentException("invalid params for Renew Token" +
-            ";t="+token+";j="+jobId+";c="+conf);
-      }
-    }
-    public void setTimerTask(TimerTask tTask) {
-      timerTask = tTask;
-    }
-    @Override
-    public String toString() {
-      return token + ";exp="+expirationDate;
-    }
-    @Override
-    public boolean equals (Object obj) {
-      if (obj == this) {
-        return true;
-      } else if (obj == null || getClass() != obj.getClass()) {
-        return false;
-      } else {
-        return token.equals(((DelegationTokenToRenew)obj).token);
-      }
-    }
-    @Override
-    public int hashCode() {
-      return token.hashCode();
-    }
-  }
-  
-  // global single timer (daemon)
-  private static Timer renewalTimer = new Timer(true);
-  
-  //delegation token canceler thread
-  private static DelegationTokenCancelThread dtCancelThread =
-    new DelegationTokenCancelThread();
-  static {
-    dtCancelThread.start();
-  }
-
-  
-  //managing the list of tokens using Map
-  // jobId=>List<tokens>
-  private static Set<DelegationTokenToRenew> delegationTokens = 
-    Collections.synchronizedSet(new HashSet<DelegationTokenToRenew>());
-  
-  private static class DelegationTokenCancelThread extends Thread {
-    private static class TokenWithConf {
-      Token<?> token;
-      Configuration conf;
-      TokenWithConf(Token<?> token, Configuration conf) {
-        this.token = token;
-        this.conf = conf;
-      }
-    }
-    private LinkedBlockingQueue<TokenWithConf> queue =  
-      new LinkedBlockingQueue<TokenWithConf>();
-     
-    public DelegationTokenCancelThread() {
-      super("Delegation Token Canceler");
-      setDaemon(true);
-    }
-    public void cancelToken(Token<?> token,  
-        Configuration conf) {
-      TokenWithConf tokenWithConf = new TokenWithConf(token, conf);
-      while (!queue.offer(tokenWithConf)) {
-        LOG.warn("Unable to add token " + token + " for cancellation. " +
-        		 "Will retry..");
-        try {
-          Thread.sleep(100);
-        } catch (InterruptedException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-
-    public void run() {
-      while (true) {
-        TokenWithConf tokenWithConf = null;
-        try {
-          tokenWithConf = queue.take();
-          final TokenWithConf current = tokenWithConf;
-          
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Canceling token " + tokenWithConf.token.getService());
-          }
-          // need to use doAs so that http can find the kerberos tgt
-          UserGroupInformation.getLoginUser().doAs(
-              new PrivilegedExceptionAction<Void>() {
-
-                @Override
-                public Void run() throws Exception {
-                  current.token.cancel(current.conf);
-                  return null;
-                }
-              });
-        } catch (IOException e) {
-          LOG.warn("Failed to cancel token " + tokenWithConf.token + " " +  
-              StringUtils.stringifyException(e));
-        } catch (InterruptedException ie) {
-          return;
-        } catch (Throwable t) {
-          LOG.warn("Got exception " + StringUtils.stringifyException(t) + 
-                   ". Exiting..");
-          System.exit(-1);
-        }
-      }
-    }
-  }
-  //adding token
-  private static void addTokenToList(DelegationTokenToRenew t) {
-    delegationTokens.add(t);
-  }
-  
-  public static synchronized void registerDelegationTokensForRenewal(
-      JobID jobId, Credentials ts, Configuration conf) throws IOException {
-    if(ts==null)
-      return; //nothing to add
-    
-    Collection <Token<?>> tokens = ts.getAllTokens();
-    long now = System.currentTimeMillis();
-
-    for (Token<?> t : tokens) {
-      // first renew happens immediately
-      if (t.isManaged()) {
-        DelegationTokenToRenew dtr = new DelegationTokenToRenew(jobId, t, conf,
-            now);
-
-        addTokenToList(dtr);
-
-        setTimerForTokenRenewal(dtr, true);
-        LOG.info("registering token for renewal for service =" + t.getService()
-            + " and jobID = " + jobId);
-      }
-    }
-  }
-    
-  /**
-   * Task - to renew a token
-   *
-   */
-  private static class RenewalTimerTask extends TimerTask {
-    private DelegationTokenToRenew dttr;
-    
-    RenewalTimerTask(DelegationTokenToRenew t) {  dttr = t;  }
-    
-    @Override
-    public void run() {
-      Token<?> token = dttr.token;
-      long newExpirationDate=0;
-      try {
-        // need to use doAs so that http can find the kerberos tgt
-        dttr.expirationDate = UserGroupInformation.getLoginUser().doAs(
-            new PrivilegedExceptionAction<Long>() {
-
-              @Override
-              public Long run() throws Exception {
-                return dttr.token.renew(dttr.conf);
-              }
-            });
-
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("renewing for:" + token.getService() + ";newED="
-              + dttr.expirationDate);
-        }
-        setTimerForTokenRenewal(dttr, false);// set the next one
-      } catch (Exception e) {
-        LOG.error("Exception renewing token" + token + ". Not rescheduled", e);
-        removeFailedDelegationToken(dttr);
-      }
-    }
-  }
-  
-  /**
-   * find the soonest expiring token and set it for renew
-   */
-  private static void setTimerForTokenRenewal(
-      DelegationTokenToRenew token, boolean firstTime) {
-      
-    // calculate timer time
-    long now = System.currentTimeMillis();
-    long renewIn;
-    if(firstTime) {
-      renewIn = now;
-    } else {
-      long expiresIn = (token.expirationDate - now); 
-      renewIn = now + expiresIn - expiresIn/10; // little before expiration
-    }
-    
-    // need to create new timer every time
-    TimerTask tTask = new RenewalTimerTask(token);
-    token.setTimerTask(tTask); // keep reference to the timer
-
-    renewalTimer.schedule(token.timerTask, new Date(renewIn));
-  }
-
-  /**
-   * removing all tokens renewals
-   */
-  static public void close() {
-    renewalTimer.cancel();
-    delegationTokens.clear();
-  }
-  
-  // cancel a token
-  private static void cancelToken(DelegationTokenToRenew t) {
-    dtCancelThread.cancelToken(t.token, t.conf);
-  }
-  
-  /**
-   * removing failed DT
-   * @param jobId
-   */
-  private static void removeFailedDelegationToken(DelegationTokenToRenew t) {
-    JobID jobId = t.jobId;
-    if (LOG.isDebugEnabled())
-      LOG.debug("removing failed delegation token for jobid=" + jobId + 
-          ";t=" + t.token.getService());
-    delegationTokens.remove(t);
-    // cancel the timer
-    if(t.timerTask!=null)
-      t.timerTask.cancel();
-  }
-  
-  /**
-   * removing DT for completed jobs
-   * @param jobId
-   */
-  public static void removeDelegationTokenRenewalForJob(JobID jobId) {
-    synchronized (delegationTokens) {
-      Iterator<DelegationTokenToRenew> it = delegationTokens.iterator();
-      while(it.hasNext()) {
-        DelegationTokenToRenew dttr = it.next();
-        if (dttr.jobId.equals(jobId)) {
-          if (LOG.isDebugEnabled())
-            LOG.debug("removing delegation token for jobid=" + jobId + 
-                ";t=" + dttr.token.getService());
-
-          // cancel the timer
-          if(dttr.timerTask!=null)
-            dttr.timerTask.cancel();
-
-          // cancel the token
-          cancelToken(dttr);
-
-          it.remove();
-        }
-      }
-    }
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/TestDelegationTokenRenewal.java

@@ -1,330 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.security.token;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.security.token.TokenRenewer;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * unit test - 
- * tests addition/deletion/cancelation of renewals of delegation tokens
- *
- */
-@Ignore
-public class TestDelegationTokenRenewal {
-  private static final Log LOG = 
-      LogFactory.getLog(TestDelegationTokenRenewal.class);
-
-  private static final Text KIND = 
-    new Text("TestDelegationTokenRenewal.Token");
-
-  public static class Renewer extends TokenRenewer {
-    private static int counter = 0;
-    private static Token<?> lastRenewed = null;
-    private static Token<?> tokenToRenewIn2Sec = null;
-
-    @Override
-    public boolean handleKind(Text kind) {
-      return KIND.equals(kind);
-    }
-
-    @Override
-    public boolean isManaged(Token<?> token) throws IOException {
-      return true;
-    }
-
-    @Override
-    public long renew(Token<?> t, Configuration conf) throws IOException {
-      MyToken token = (MyToken)t;
-      if(token.isCanceled()) {
-        throw new InvalidToken("token has been canceled");
-      }
-      lastRenewed = token;
-      counter ++;
-      LOG.info("Called MYDFS.renewdelegationtoken " + token + 
-               ";this dfs=" + this.hashCode() + ";c=" + counter);
-      if(tokenToRenewIn2Sec == token) { 
-        // this token first renewal in 2 seconds
-        LOG.info("RENEW in 2 seconds");
-        tokenToRenewIn2Sec=null;
-        return 2*1000 + System.currentTimeMillis();
-      } else {
-        return 86400*1000 + System.currentTimeMillis();
-      }
-    }
-
-    @Override
-    public void cancel(Token<?> t, Configuration conf) {
-      MyToken token = (MyToken)t;
-      LOG.info("Cancel token " + token);
-      token.cancelToken();
-   }
-
-  }
-
-  private static Configuration conf;
- 
-  @BeforeClass
-  public static void setUp() throws Exception {
-    conf = new Configuration();
-    
-    // create a fake FileSystem (MyFS) and assosiate it
-    // with "hdfs" schema.
-    URI uri = new URI(DelegationTokenRenewal.SCHEME+"://localhost:0");
-    System.out.println("scheme is : " + uri.getScheme());
-    conf.setClass("fs." + uri.getScheme() + ".impl", MyFS.class, DistributedFileSystem.class);
-    FileSystem.setDefaultUri(conf, uri);
-    LOG.info("filesystem uri = " + FileSystem.getDefaultUri(conf).toString());
-  }
-  
-  private static class MyDelegationTokenSecretManager extends DelegationTokenSecretManager {
-
-    public MyDelegationTokenSecretManager(long delegationKeyUpdateInterval,
-        long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
-        long delegationTokenRemoverScanInterval, FSNamesystem namesystem) {
-      super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
-          delegationTokenRenewInterval, delegationTokenRemoverScanInterval,
-          namesystem);
-    }
-    
-    @Override //DelegationTokenSecretManager
-    public void logUpdateMasterKey(DelegationKey key) throws IOException {
-      return;
-    }
-  }
-  
-  /**
-   * add some extra functionality for testing
-   * 1. toString();
-   * 2. cancel() and isCanceled()
-   */
-  private static class MyToken extends Token<DelegationTokenIdentifier> {
-    public String status = "GOOD";
-    public static final String CANCELED = "CANCELED";
-
-    public MyToken(DelegationTokenIdentifier dtId1,
-        MyDelegationTokenSecretManager sm) {
-      super(dtId1, sm);
-      setKind(KIND);
-      status = "GOOD";
-    }
-    
-    public boolean isCanceled() {return status.equals(CANCELED);}
-
-    public void cancelToken() {this.status=CANCELED;}
-
-    public String toString() {
-      StringBuilder sb = new StringBuilder(1024);
-      
-      sb.append("id=");
-      String id = StringUtils.byteToHexString(this.getIdentifier());
-      int idLen = id.length();
-      sb.append(id.substring(idLen-6));
-      sb.append(";k=");
-      sb.append(this.getKind());
-      sb.append(";s=");
-      sb.append(this.getService());
-      return sb.toString();
-    }
-  }
-
-  /**
-   * fake FileSystem 
-   * overwrites three methods
-   * 1. getDelegationToken() - generates a token
-   * 2. renewDelegataionToken - counts number of calls, and remembers 
-   * most recently renewed token.
-   * 3. cancelToken -cancels token (subsequent renew will cause IllegalToken 
-   * exception
-   */
-  static class MyFS extends DistributedFileSystem {
-    
-    public MyFS() {}
-    public void close() {}
-    @Override
-    public void initialize(URI uri, Configuration conf) throws IOException {}
-    
-    @Override 
-    public MyToken getDelegationToken(Text renewer) throws IOException {
-      MyToken result = createTokens(renewer);
-      LOG.info("Called MYDFS.getdelegationtoken " + result);
-      return result;
-    }
-
-  }
-  
-  /**
-   * auxilary - create token
-   * @param renewer
-   * @return
-   * @throws IOException
-   */
-  static MyToken createTokens(Text renewer) 
-    throws IOException {
-    Text user1= new Text("user1");
-    
-    MyDelegationTokenSecretManager sm = new MyDelegationTokenSecretManager(
-        DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
-        DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
-        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
-        3600000, null);
-    sm.startThreads();
-    
-    DelegationTokenIdentifier dtId1 = 
-      new DelegationTokenIdentifier(user1, renewer, user1);
-    
-    MyToken token1 = new MyToken(dtId1, sm);
-    
-   
-    token1.setService(new Text("localhost:0"));
-    return token1;
-  }
-  
-  
-  /**
-   * Basic idea of the test:
-   * 1. create tokens.
-   * 2. Mark one of them to be renewed in 2 seconds (istead of
-   * 24 hourse)
-   * 3. register them for renewal
-   * 4. sleep for 3 seconds
-   * 5. count number of renewals (should 3 initial ones + one extra)
-   * 6. register another token for 2 seconds 
-   * 7. cancel it immediately
-   * 8. Sleep and check that the 2 seconds renew didn't happen 
-   * (totally 5 reneals)
-   * 9. check cancelation
-   * @throws IOException
-   * @throws URISyntaxException
-   */
-  @Test
-  public void testDTRenewal () throws Exception {
-    MyFS dfs = (MyFS)FileSystem.get(conf);
-    LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());
-    // Test 1. - add three tokens - make sure exactly one get's renewed
-    
-    // get the delegation tokens
-    MyToken token1, token2, token3;
-    token1 = dfs.getDelegationToken(new Text("user1"));
-    token2 = dfs.getDelegationToken(new Text("user2"));
-    token3 = dfs.getDelegationToken(new Text("user3"));
-
-    //to cause this one to be set for renew in 2 secs
-    Renewer.tokenToRenewIn2Sec = token1;
-    LOG.info("token="+token1+" should be renewed for 2 secs");
-    
-    // two distinct Namenodes
-    String nn1 = DelegationTokenRenewal.SCHEME + "://host1:0";
-    String nn2 = DelegationTokenRenewal.SCHEME + "://host2:0";
-    String nn3 = DelegationTokenRenewal.SCHEME + "://host3:0";
-    
-    Credentials ts = new Credentials();
-    
-    // register the token for renewal
-    ts.addToken(new Text(nn1), token1);
-    ts.addToken(new Text(nn2), token2);
-    ts.addToken(new Text(nn3), token3);
-    
-    // register the tokens for renewal
-    DelegationTokenRenewal.registerDelegationTokensForRenewal(
-        new JobID("job1", 1), ts, conf);
-    // first 3 initial renewals + 1 real
-    int numberOfExpectedRenewals = 3+1; 
-    
-    int attempts = 10;
-    while(attempts-- > 0) {
-      try {
-        Thread.sleep(3*1000); // sleep 3 seconds, so it has time to renew
-      } catch (InterruptedException e) {}
-      
-      // since we cannot guarantee timely execution - let's give few chances
-      if(Renewer.counter==numberOfExpectedRenewals)
-        break;
-    }
-    
-    assertEquals("renew wasn't called as many times as expected(4):",
-        numberOfExpectedRenewals, Renewer.counter);
-    assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, 
-        token1);
-    
-    // Test 2. 
-    // add another token ( that expires in 2 secs). Then remove it, before
-    // time is up.
-    // Wait for 3 secs , and make sure no renews were called
-    ts = new Credentials();
-    MyToken token4 = dfs.getDelegationToken(new Text("user4"));
-    
-    //to cause this one to be set for renew in 2 secs
-    Renewer.tokenToRenewIn2Sec = token4; 
-    LOG.info("token="+token4+" should be renewed for 2 secs");
-    
-    String nn4 = DelegationTokenRenewal.SCHEME + "://host4:0";
-    ts.addToken(new Text(nn4), token4);
-    
-
-    JobID jid2 = new JobID("job2",1);
-    DelegationTokenRenewal.registerDelegationTokensForRenewal(jid2, ts, conf);
-    DelegationTokenRenewal.removeDelegationTokenRenewalForJob(jid2);
-    numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
-    try {
-      Thread.sleep(6*1000); // sleep 6 seconds, so it has time to renew
-    } catch (InterruptedException e) {}
-    System.out.println("Counter = " + Renewer.counter + ";t="+ 
-                       Renewer.lastRenewed);
-    
-    // counter and the token should stil be the old ones
-    assertEquals("renew wasn't called as many times as expected",
-        numberOfExpectedRenewals, Renewer.counter);
-    
-    // also renewing of the cancelled token should fail
-    try {
-      token4.renew(conf);
-      fail("Renew of canceled token didn't fail");
-    } catch (InvalidToken ite) {
-      //expected
-    }
-  }
-}