commit 94e20361420f8fcb65bed1c8a66495d248675c06

Author: Owen O'Malley <omalley@apache.org>
Date:   Thu Apr 22 23:32:15 2010 -0700

    HADOOP-6670. Use the UserGroupInformation's Subject as the criteria for
    equals and hashCode. (omalley)
    
    This reverts commit 6413d7d0b936474ddf355e1e4cc76d16ab773976.
    
    Conflicts:
    
    	YAHOO-CHANGES.txt
    
    +++ b/YAHOO-CHANGES.txt
    +    HADOOP-6670. Use the UserGroupInformation's Subject as the criteria for
    +    equals and hashCode. (omalley)
    +
    +    task cleanup attempt's log directory. (Amareshwari Sreeramadasu via
    +    vinodkv)
    +    MAPREDUCE-1533. JobTracker performance enhancements. (Amar Kamat via
    +    vinodkv)
    +    HDFS-1011. Improve Logging in HDFSProxy to include cluster name associated
    +    with the request (Srikanth Sundarrajan via Nicholas)
    +    HDFS-1010. Retrieve group information from UnixUserGroupInformation
    +    instead of LdapEntry (Srikanth Sundarrajan via Nicholas)
    +    HADOOP-6687.  user object in the subject in UGI should be reused in case
    +    of a relogin. (jitendra)
    +    HADOOP-5647. TestJobHistory fails if /tmp/_logs is not writable to.
    +    Testcase should not depend on /tmp. (Ravi Gummadi via vinodkv)
    +    MAPREDUCE-181. Bug fix for Secure job submission. (Ravi Gummadi via
    +    vinodkv)
    +    MAPREDUCE-1526. Cache the job related information while submitting the
    +    job. (rksingh)
    +    HDFS-1005. Fsck security. Makes it work over kerberized SSL (boryas and
    +    jhoman)
    +    HADOOP-6600,HDFS-1003,MAPREDUCE-1539. mechanism for authorization check
    +    for inter-server protocols(boryas)
    +
    +    HADOOP-6580,HDFS-993,MAPREDUCE-1516. UGI should contain authentication
    +    method.
    +    Namenode and JT should issue a delegation token only for kerberos
    +    authenticated  clients. (jitendra)
    +    HDFS-984,HADOOP-6573,MAPREDUCE-1537. Delegation Tokens should be persisted
    +    in Namenode, and corresponding changes in common and mr. (jitendra)
    +    HADOOP-6586. Log authentication and authorization failures and successes
    +    (boryas)
    +
    +x


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077420 13f79535-47bb-0310-9956-ffa450edef68

src/core/org/apache/hadoop/fs/FileSystem.java

@@ -228,7 +228,10 @@ public abstract class FileSystem extends Configured implements Closeable {
    * @throws IOException
    */
   public static void closeAll() throws IOException {
+    LOG.debug("Starting clear of FileSystem cache with " + CACHE.size() +
+              " elements.");
     CACHE.closeAll();
+    LOG.debug("Done clearing cache");
   }
 
   /** Make sure that a path specifies a FileSystem. */
@@ -1257,6 +1260,7 @@ public abstract class FileSystem extends Configured implements Closeable {
     // delete all files that were marked as delete-on-exit.
     processDeleteOnExit();
     CACHE.remove(this.key, this);
+    LOG.debug("Removing filesystem for " + getUri());
   }
 
   /** Return the total size of all files in the filesystem.*/
@@ -1382,6 +1386,7 @@ public abstract class FileSystem extends Configured implements Closeable {
   private static FileSystem createFileSystem(URI uri, Configuration conf
       ) throws IOException {
     Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
+    LOG.debug("Creating filesystem for " + uri);
     if (clazz == null) {
       throw new IOException("No FileSystem for scheme: " + uri.getScheme());
     }
@@ -1485,6 +1490,14 @@ public abstract class FileSystem extends Configured implements Closeable {
         return "("+ugi.toString() + ")@" + scheme + "://" + authority;        
       }
     }
+    
+    /**
+     * Get the number of file systems in the cache.
+     * @return the number of cached file systems
+     */
+    int size() {
+      return map.size();
+    }
   }
   
   public static final class Statistics {

src/core/org/apache/hadoop/ipc/Client.java

@@ -1050,8 +1050,8 @@ public class Client {
      if (obj instanceof ConnectionId) {
        ConnectionId id = (ConnectionId) obj;
        return address.equals(id.address) && protocol == id.protocol && 
-              ticket == id.ticket;
-       //Note : ticket is a ref comparision.
+              ((ticket != null && ticket.equals(id.ticket)) ||
+               (ticket == id.ticket));
      }
      return false;
     }
@@ -1059,7 +1059,7 @@ public class Client {
     @Override
     public int hashCode() {
       return (address.hashCode() + PRIME * System.identityHashCode(protocol)) ^ 
-             System.identityHashCode(ticket);
+             (ticket == null ? 0 : ticket.hashCode());
     }
   }  
 }

src/core/org/apache/hadoop/security/UserGroupInformation.java

@@ -935,7 +935,7 @@ public class UserGroupInformation {
     } else if (o == null || getClass() != o.getClass()) {
       return false;
     } else {
-      return subject.equals(((UserGroupInformation) o).subject);
+      return subject == ((UserGroupInformation) o).subject;
     }
   }
 
@@ -944,7 +944,7 @@ public class UserGroupInformation {
    */
   @Override
   public int hashCode() {
-    return subject.hashCode();
+    return System.identityHashCode(subject);
   }
 
   /**

src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java

@@ -297,18 +297,57 @@ public class TestTrackerDistributedCacheManager extends TestCase {
     if (!canRun()) {
       return;
     }
-    checkLocalizedPath("true");
-    checkLocalizedPath("false");
+    checkLocalizedPath(true);
+    checkLocalizedPath(false);
   }
   
-  private void checkLocalizedPath(String visibility) 
+  private void appendStringArray(StringBuilder buffer, String[] data) {
+    if (data != null && data.length != 0) {
+      buffer.append(data[0]);
+      for(int i=1; i < data.length; i++) {
+        buffer.append(',');
+        buffer.append(data[i]);
+      }
+    }
+  }
+
+  private void appendUriArray(StringBuilder buffer, URI[] data) {
+    if (data != null && data.length != 0) {
+      buffer.append(data[0]);
+      for(int i=1; i < data.length; i++) {
+        buffer.append(',');
+        buffer.append(data[i]);
+      }
+    }
+  }
+
+  private void dumpState(Configuration conf1) throws IOException {
+    StringBuilder buf = new StringBuilder();
+    buf.append("\nFiles:");
+    appendUriArray(buf, DistributedCache.getCacheFiles(conf1));
+    buf.append("\nArchives:");
+    appendUriArray(buf, DistributedCache.getCacheArchives(conf1));
+    buf.append("\nFile Visible:");
+    appendStringArray(buf, TrackerDistributedCacheManager.getFileVisibilities
+                      (conf1));
+    buf.append("\nArchive Visible:");
+    appendStringArray(buf, TrackerDistributedCacheManager.getArchiveVisibilities
+                      (conf1));
+    buf.append("\nFile timestamps:");
+    appendStringArray(buf, DistributedCache.getFileTimestamps(conf1));
+    buf.append("\nArchive timestamps:");
+    appendStringArray(buf, DistributedCache.getArchiveTimestamps(conf1));
+    LOG.info("state = " + buf.toString());
+  }
+  
+  private void checkLocalizedPath(boolean visibility) 
   throws IOException, LoginException, InterruptedException {
     TrackerDistributedCacheManager manager = 
       new TrackerDistributedCacheManager(conf, taskController);
     String userName = getJobOwnerName();
     File workDir = new File(TEST_ROOT_DIR, "workdir");
     Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile");
-    if ("true".equals(visibility)) {
+    if (visibility) {
       createPublicTempFile(cacheFile);
     } else {
       createPrivateTempFile(cacheFile);
@@ -319,6 +358,7 @@ public class TestTrackerDistributedCacheManager extends TestCase {
     DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
     TrackerDistributedCacheManager.determineTimestamps(conf1);
     TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+    dumpState(conf1);
 
     // Task localizing for job
     TaskDistributedCacheManager handle = manager
@@ -328,7 +368,7 @@ public class TestTrackerDistributedCacheManager extends TestCase {
           TaskTracker.getPublicDistributedCacheDir());
     TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0);
     String distCacheDir;
-    if ("true".equals(visibility)) {
+    if (visibility) {
       distCacheDir = TaskTracker.getPublicDistributedCacheDir(); 
     } else {
       distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName);
@@ -337,19 +377,19 @@ public class TestTrackerDistributedCacheManager extends TestCase {
       manager.getLocalCache(cacheFile.toUri(), conf1, distCacheDir,
           fs.getFileStatus(cacheFile), false,
           c.timestamp, new Path(TEST_ROOT_DIR), false,
-          Boolean.parseBoolean(visibility));
+          visibility);
     assertTrue("Cache file didn't get localized in the expected directory. " +
         "Expected localization to happen within " + 
         ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir +
         ", but was localized at " + 
         localizedPath, localizedPath.toString().contains(distCacheDir));
-    if ("true".equals(visibility)) {
+    if (visibility) {
       checkPublicFilePermissions(new Path[]{localizedPath});
     } else {
       checkFilePermissions(new Path[]{localizedPath});
     }
   }
-  
+
   /**
    * Check proper permissions on the cache files
    * 

src/test/org/apache/hadoop/fs/TestFileSystem.java

@@ -647,28 +647,26 @@ public class TestFileSystem extends TestCase {
     assertNotSame(fsA, fsB);
     
     Token<T> t1 = mock(Token.class);
-    ugiA = UserGroupInformation.createRemoteUser("foo");
-    ugiA.addToken(t1);
+    UserGroupInformation ugiA2 = UserGroupInformation.createRemoteUser("foo");
     
-    fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+    fsA = ugiA2.doAs(new PrivilegedExceptionAction<FileSystem>() {
       public FileSystem run() throws Exception {
         return FileSystem.get(new URI("cachedfile://a"), conf);
       }
     });
-    //Although the users in the UGI are same, ugiA has tokens in it, and
-    //we should end up with different filesystems corresponding to the two UGIs
+    // Although the users in the UGI are same, they have different subjects
+    // and so are different.
     assertNotSame(fsA, fsA1);
     
-    ugiA = UserGroupInformation.createRemoteUser("foo");
     ugiA.addToken(t1);
     
-    fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+    fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
       public FileSystem run() throws Exception {
         return FileSystem.get(new URI("cachedfile://a"), conf);
       }
     });
-    //Now the users in the UGI are the same, and they also have the same token.
-    //We should have the same filesystem for both
+    // Make sure that different UGI's with the same subject lead to the same
+    // file system.
     assertSame(fsA, fsA1);
   }
 }

src/test/org/apache/hadoop/security/TestUserGroupInformation.java

@@ -156,10 +156,17 @@ public class TestUserGroupInformation {
       UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
 
     assertEquals(uugi, uugi);
-    // The subjects should be equal, so this should work
-    assertTrue(uugi.equals(
-                 UserGroupInformation.createUserForTesting
-                   (USER_NAME, GROUP_NAMES)));
+    // The subjects should be different, so this should fail
+    UserGroupInformation ugi2 = 
+      UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
+    assertFalse(uugi.equals(ugi2));
+    assertFalse(uugi.hashCode() == ugi2.hashCode());
+
+    // two ugi that have the same subject need to be equal
+    UserGroupInformation ugi3 = new UserGroupInformation(uugi.getSubject());
+    assertEquals(uugi, ugi3);
+    assertEquals(uugi.hashCode(), ugi3.hashCode());
+    
     // ensure that different UGI with the same subject are equal
     assertEquals(uugi, new UserGroupInformation(uugi.getSubject()));
   }
@@ -172,8 +179,8 @@ public class TestUserGroupInformation {
         "RealUser", GROUP_NAMES);
     UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser(
         USER_NAME, realUgi1);
-    UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
-        USER_NAME, realUgi2);
+    UserGroupInformation proxyUgi2 =
+      new UserGroupInformation( proxyUgi1.getSubject());
     UserGroupInformation remoteUgi = UserGroupInformation.createRemoteUser(USER_NAME);
     assertEquals(proxyUgi1, proxyUgi2);
     assertFalse(remoteUgi.equals(proxyUgi1));
@@ -284,16 +291,16 @@ public class TestUserGroupInformation {
         return null;
       }
     });
-    UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
-        "proxy", ugi);
+    UserGroupInformation proxyUgi2 = 
+      new UserGroupInformation(proxyUgi.getSubject());
     proxyUgi2.setAuthenticationMethod(AuthenticationMethod.PROXY);
     Assert.assertEquals(proxyUgi, proxyUgi2);
     // Equality should work if authMethod is null
     UserGroupInformation realugi = UserGroupInformation.getCurrentUser();
     UserGroupInformation proxyUgi3 = UserGroupInformation.createProxyUser(
         "proxyAnother", realugi);
-    UserGroupInformation proxyUgi4 = UserGroupInformation.createProxyUser(
-        "proxyAnother", realugi);
+    UserGroupInformation proxyUgi4 = 
+      new UserGroupInformation(proxyUgi3.getSubject());
     Assert.assertEquals(proxyUgi3, proxyUgi4);
   }
 }