HADOOP-2381. Support permission information in FileStatus. Client

Protocol version changed from 21 to 22.  (Raghu Angadi via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@604340 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -33,6 +33,9 @@ Trunk (unreleased changes)
     HADOOP-2401.  Only the current leaseholder can abandon a block for
     a HDFS file.  ClientProtocol version changed from 20 to 21.
     (Tsz Wo (Nicholas), SZE via dhruba)
+
+    HADOOP-2381.  Support permission information in FileStatus. Client
+    Protocol version changed from 21 to 22.  (Raghu Angadi via dhruba)
     
   NEW FEATURES
 

src/java/org/apache/hadoop/dfs/ClientProtocol.java

@@ -37,8 +37,9 @@ interface ClientProtocol extends VersionedProtocol {
    * 19 : rollEditLog() returns a token to uniquely identify the editfile.
    * 20 : getContentLength returns the total size in bytes of a directory subtree
    * 21 : add lease holder as a parameter in abandonBlock(...)
+   * 22 : Serialization of FileStatus has changed.
    */
-  public static final long versionID = 21L;
+  public static final long versionID = 22L;
   
   ///////////////////////////////////////
   // File contents

src/java/org/apache/hadoop/fs/FileStatus.java

@@ -21,6 +21,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 
@@ -34,22 +35,33 @@ public class FileStatus implements Writable {
   private short block_replication;
   private long blocksize;
   private long modification_time;
-
-  public FileStatus() { this(0, false, 0, 0, 0, null); }
-
+  private FsPermission permission;
+  private String owner;
+  private String group;
+  
+  public FileStatus() { this(0, false, 0, 0, 0, null, null, null, null); }
+  
+  //We should deprecate this soon?
   public FileStatus(long length, boolean isdir, int block_replication,
-             long blocksize, long modification_time, Path path) {
-    this(length, isdir, (short)block_replication, blocksize,
-         modification_time, path);
-  }
+                    long blocksize, long modification_time, Path path) {
 
-  public FileStatus(long length, boolean isdir, short block_replication,
-             long blocksize, long modification_time, Path path) {
+    this(length, isdir, block_replication, blocksize, modification_time,
+         null, null, null, path);
+  }
+  
+  public FileStatus(long length, boolean isdir, int block_replication,
+                    long blocksize, long modification_time,
+                    FsPermission permission, String owner, String group, 
+                    Path path) {
     this.length = length;
     this.isdir = isdir;
-    this.block_replication = block_replication;
+    this.block_replication = (short)block_replication;
     this.blocksize = blocksize;
     this.modification_time = modification_time;
+    this.permission = (permission == null) ? 
+                      FsPermission.getDefault() : permission;
+    this.owner = (owner == null) ? "" : owner;
+    this.group = (group == null) ? "" : group;
     this.path = path;
   }
 
@@ -92,10 +104,69 @@ public class FileStatus implements Writable {
     return modification_time;
   }
 
+  /**
+   * Get FsPermission associated with the file.
+   * @return permssion. If a filesystem does not have a notion of permissions
+   *         or if permissions could not be determined, then default 
+   *         permissions equivalent of "rwxrwxrwx" is returned.
+   */
+  public FsPermission getPermission() {
+    return permission;
+  }
+  
+  /**
+   * Get the owner of the file.
+   * @return owner of the file. The string could be empty if there is no
+   *         notion of owner of a file in a filesystem or if it could not 
+   *         be determined (rare).
+   */
+  public String getOwner() {
+    return owner;
+  }
+  
+  /**
+   * Get the group associated with the file.
+   * @return group for the file. The string could be empty if there is no
+   *         notion of group of a file in a filesystem or if it could not 
+   *         be determined (rare).
+   */
+  public String getGroup() {
+    return group;
+  }
+  
   public Path getPath() {
     return path;
   }
 
+  /* These are provided so that these values could be loaded lazily 
+   * by a filesystem (e.g. local file system).
+   */
+  
+  /**
+   * Sets permission.
+   * @param permission if permission is null, default value is set
+   */
+  protected void setPermission(FsPermission permission) {
+    this.permission = (permission == null) ? 
+                      FsPermission.getDefault() : permission;
+  }
+  
+  /**
+   * Sets owner.
+   * @param owner if it is null, default value is set
+   */  
+  protected void setOwner(String owner) {
+    this.owner = (owner == null) ? "" : owner;
+  }
+  
+  /**
+   * Sets group.
+   * @param group if it is null, default value is set
+   */  
+  protected void setGroup(String group) {
+    this.group = (group == null) ? "" :  group;
+  }
+  
   //////////////////////////////////////////////////
   // Writable
   //////////////////////////////////////////////////
@@ -106,6 +177,9 @@ public class FileStatus implements Writable {
     out.writeShort(block_replication);
     out.writeLong(blocksize);
     out.writeLong(modification_time);
+    permission.write(out);
+    Text.writeString(out, owner);
+    Text.writeString(out, group);
   }
 
   public void readFields(DataInput in) throws IOException {
@@ -116,6 +190,9 @@ public class FileStatus implements Writable {
     this.block_replication = in.readShort();
     blocksize = in.readLong();
     modification_time = in.readLong();
+    permission.readFields(in);
+    owner = Text.readString(in);
+    group = Text.readString(in);
   }
 
 }

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -387,7 +387,7 @@ public abstract class FileSystem extends Configured {
                                             long blockSize,
                                             Progressable progress
                                             ) throws IOException {
-    return this.create(f, FsPermission.getDefault(getConf()),
+    return this.create(f, FsPermission.getDefault(),
         overwrite, bufferSize, replication, blockSize, progress);
   }
 
@@ -836,7 +836,7 @@ public abstract class FileSystem extends Configured {
    * Call {@link #mkdirs(Path, FsPermission)} with default permission.
    */
   public boolean mkdirs(Path f) throws IOException {
-    return mkdirs(f, FsPermission.getDefault(getConf()));
+    return mkdirs(f, FsPermission.getDefault());
   }
 
   /**

src/java/org/apache/hadoop/fs/FsShell.java

@@ -551,9 +551,11 @@ public class FsShell extends Configured implements Tool {
     }
   }
 
-  /* list all files under the directory <i>src</i>*/
+  /* list all files under the directory <i>src</i>
+   * ideally we should provide "-l" option, that lists like "ls -l".
+   */
   private void ls(Path src, boolean recursive, boolean printHeader) throws IOException {
-    Path items[] = fs.listPaths(src);
+    FileStatus items[] = fs.listStatus(src);
     if (items == null) {
       throw new IOException("Could not get listing for " + src);
     } else {
@@ -561,15 +563,18 @@ public class FsShell extends Configured implements Tool {
         System.out.println("Found " + items.length + " items");
       }
       for (int i = 0; i < items.length; i++) {
-        Path cur = items[i];
-        FileStatus stat = fs.getFileStatus(cur);
+        FileStatus stat = items[i];
+        Path cur = stat.getPath();
         String mdate = dateForm.format(new Date(stat.getModificationTime()));
         System.out.println(cur.toUri().getPath() + "\t" 
                            + (stat.isDir() ? 
                               "<dir>\t" : 
                               ("<r " + stat.getReplication() 
                                + ">\t" + stat.getLen()))
-                           + "\t" + mdate);
+                           + "\t" + mdate 
+                           + "\t" + stat.getPermission()
+                           + "\t" + stat.getOwner() 
+                           + "\t" + stat.getGroup());
         if (recursive && stat.isDir()) {
           ls(cur, recursive, printHeader);
         }

src/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -357,36 +357,78 @@ public class RawLocalFileSystem extends FileSystem {
   }
 
   static class RawLocalFileStatus extends FileStatus {
-    private File file;
-    private PermissionStatus permissions;
-
+    /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
+     * We recognize if the information is already loaded by check if
+     * onwer.equals("").
+     */
+    private boolean isPermissionLoaded() {
+      return !super.getOwner().equals(""); 
+    }
+    
     RawLocalFileStatus(File f, long defaultBlockSize) {
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
             f.lastModified(), new Path(f.toURI().toString()));
-      file = f;
+    }
+    
+    @Override
+    public FsPermission getPermission() {
+      if (!isPermissionLoaded()) {
+        loadPermissionInfo();
+      }
+      return super.getPermission();
     }
 
-    PermissionStatus getPermissionStatus() {
-      if (permissions == null) {
-        try {
-          permissions = getPermissionStatus(file);
-        }
-        catch(IOException e) {
-          LOG.debug(StringUtils.stringifyException(e));
+    @Override
+    public String getOwner() {
+      if (!isPermissionLoaded()) {
+        loadPermissionInfo();
+      }
+      return super.getOwner();
+    }
+
+    @Override
+    public String getGroup() {
+      if (isPermissionLoaded()) {
+        loadPermissionInfo();
+      }
+      return super.getGroup();
+    }
+
+    /// loads permissions, owner, and group from `ls -ld`
+    private void loadPermissionInfo() {
+      try {
+        StringTokenizer t = new StringTokenizer(
+            execCommand(new File(getPath().toUri()), 
+                        ShellCommand.getGET_PERMISSION_COMMAND()));
+        //expected format
+        //-rw-------    1 username groupname ...
+        setPermission(FsPermission.valueOf(t.nextToken()));
+        t.nextToken();
+        setOwner(t.nextToken());
+        setGroup(t.nextToken());
+      } catch (IOException e) {
+        if (e.getMessage().contains("No such file or directory")) {                                    
+          /* XXX This is a temporary hack till HADOOP-2344 goes in.
+           * will fix it soon.
+           */
+          setPermission(null);
+          setOwner(null);
+          setGroup(null);
+        } else {
+          //this is not expected
+          throw new RuntimeException("Error while running command to get " +
+                                     "file permissions : " + 
+                                     StringUtils.stringifyException(e));
         }
       }
-      return permissions;
     }
 
-    private static PermissionStatus getPermissionStatus(File f
-        ) throws IOException {
-      StringTokenizer t = new StringTokenizer(
-          execCommand(f, ShellCommand.getGET_PERMISSION_COMMAND()));
-      //expected format
-      //-rw-------    1 username groupname ...
-      FsPermission p = FsPermission.valueOf(t.nextToken());
-      t.nextToken();
-      return new PermissionStatus(t.nextToken(), t.nextToken(), p);
+    @Override
+    public void write(DataOutput out) throws IOException {
+      if (!isPermissionLoaded()) {
+        loadPermissionInfo();
+      }
+      super.write(out);
     }
   }
 
@@ -420,9 +462,7 @@ public class RawLocalFileSystem extends FileSystem {
     String[] args = new String[cmd.length + 1];
     System.arraycopy(cmd, 0, args, 0, cmd.length);
     args[cmd.length] = f.getCanonicalPath();
-    LOG.debug("args=" + Arrays.asList(args));
     String output = ShellCommand.execCommand(args);
-    LOG.debug("output=" + output);
     return output;
   }
 }

src/java/org/apache/hadoop/fs/permission/FsPermission.java

@@ -146,14 +146,9 @@ public class FsPermission implements Writable {
     conf.setInt(UMASK_LABEL, umask.toShort());
   }
 
-  private static final FsPermission NONE = new FsPermission((short)0);
-
-  /**
-   * Get the default permission from conf.
-   * @param conf
-   */
-  public static FsPermission getDefault(Configuration conf) {
-    return NONE.applyUMask(getUMask(conf));
+  /** Get the default permission. */
+  public static FsPermission getDefault() {
+    return new FsPermission((short)0777);
   }
 
   /**
@@ -169,7 +164,7 @@ public class FsPermission implements Writable {
     for(int i = 1; i < unixSymbolicPermission.length(); i++) {
       n = n << 1;
       char c = unixSymbolicPermission.charAt(i);
-      n += c == '-' || c == 'T'? 0: 1;
+      n += (c == '-' || c == 'T' || c == 'S') ? 0: 1;
     }
     return new FsPermission((short)n);
   }

src/test/org/apache/hadoop/fs/TestLocalFileSystemPermission.java

@@ -93,9 +93,7 @@ public class TestLocalFileSystemPermission extends TestCase {
   }
 
   FsPermission getPermission(LocalFileSystem fs, Path p) throws IOException {
-    RawLocalFileSystem.RawLocalFileStatus s
-      = (RawLocalFileSystem.RawLocalFileStatus)fs.getFileStatus(p);
-    return s.getPermissionStatus().getPermission();
+    return fs.getFileStatus(p).getPermission();
   }
 
   /** Test LocalFileSystem.setOwner */
@@ -148,8 +146,6 @@ public class TestLocalFileSystemPermission extends TestCase {
   }
 
   String getGroup(LocalFileSystem fs, Path p) throws IOException {
-    RawLocalFileSystem.RawLocalFileStatus s
-      = (RawLocalFileSystem.RawLocalFileStatus)fs.getFileStatus(p);
-    return s.getPermissionStatus().getGroupName();
+    return fs.getFileStatus(p).getGroup();
   }
 }