HDFS-578. Add support for new FileSystem method for clients to get server defaults. Contributed by Kan Zhang.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/trunk@812601 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -22,6 +22,9 @@ Trunk (unreleased changes)
     HDFS-492. Add two JSON JSP pages to the Namenode for providing corrupt
     blocks/replicas information.  (Bill Zeller via szetszwo)
 
+    HDFS-578. Add support for new FileSystem method for clients to get server
+    defaults. (Kan Zhang via suresh)
+
   IMPROVEMENTS
 
     HDFS-381. Remove blocks from DataNode maps when corresponding file

src/java/org/apache/hadoop/hdfs/DFSClient.java

@@ -67,6 +67,7 @@ import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FSOutputSummer;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.FsStatus;
 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
 import org.apache.hadoop.fs.Path;
@@ -125,12 +126,15 @@ import org.apache.hadoop.util.StringUtils;
  ********************************************************/
 public class DFSClient implements FSConstants, java.io.Closeable {
   public static final Log LOG = LogFactory.getLog(DFSClient.class);
+  public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; // 1 hour
   public static final int MAX_BLOCK_ACQUIRE_FAILURES = 3;
   private static final int TCP_WINDOW_SIZE = 128 * 1024; // 128 KB
   private final ClientProtocol namenode;
   private final ClientProtocol rpcNamenode;
   final UnixUserGroupInformation ugi;
   volatile boolean clientRunning = true;
+  private volatile FsServerDefaults serverDefaults;
+  private volatile long serverDefaultsLastUpdate;
   Random r = new Random();
   final String clientName;
   final LeaseChecker leasechecker = new LeaseChecker();
@@ -328,6 +332,18 @@ public class DFSClient implements FSConstants, java.io.Closeable {
     }
   }
 
+  /**
+   * Get server default values for a number of configuration params.
+   */
+  public FsServerDefaults getServerDefaults() throws IOException {
+    long now = System.currentTimeMillis();
+    if (now - serverDefaultsLastUpdate > SERVER_DEFAULTS_VALIDITY_PERIOD) {
+      serverDefaults = namenode.getServerDefaults();
+      serverDefaultsLastUpdate = now;
+    }
+    return serverDefaults;
+  }
+
   /**
    * Report corrupt blocks that were discovered by the client.
    */

src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -434,6 +434,12 @@ public class DistributedFileSystem extends FileSystem {
     dfs.metaSave(pathname);
   }
 
+  /** {@inheritDoc} */
+  @Override
+  public FsServerDefaults getServerDefaults() throws IOException {
+    return dfs.getServerDefaults();
+  }
+
   /**
    * We need to find the blocks that didn't match.  Likely only one 
    * is corrupt but we will report both to the namenode.  In the future,

src/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
@@ -43,9 +44,9 @@ public interface ClientProtocol extends VersionedProtocol {
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 45: add create flag for create command, see Hadoop-5438
+   * 46: added a new method getServerDefaults(), see HDFS-578
    */
-  public static final long versionID = 45L;
+  public static final long versionID = 46L;
   
   ///////////////////////////////////////
   // File contents
@@ -73,6 +74,13 @@ public interface ClientProtocol extends VersionedProtocol {
                                           long offset,
                                           long length) throws IOException;
 
+  /**
+   * Get server default values for a number of configuration params.
+   * @return a set of server default configuration values
+   * @throws IOException
+   */
+  public FsServerDefaults getServerDefaults() throws IOException;
+
   /**
    * Create a new file entry in the namespace.
    * <p>

src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java

@@ -53,6 +53,10 @@ public interface FSConstants {
   public static final int SMALL_BUFFER_SIZE = Math.min(BUFFER_SIZE/2, 512);
   //TODO mb@media-style.com: should be conf injected?
   public static final long DEFAULT_BLOCK_SIZE = 64 * 1024 * 1024;
+  public static final int DEFAULT_BYTES_PER_CHECKSUM = 512;
+  public static final int DEFAULT_WRITE_PACKET_SIZE = 64 * 1024;
+  public static final short DEFAULT_REPLICATION_FACTOR = 3;
+  public static final int DEFAULT_FILE_BUFFER_SIZE = 4096;
   public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
 
   public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;

src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocat
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.*;
 import org.apache.hadoop.ipc.Server;
@@ -200,8 +201,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
   private long heartbeatExpireInterval;
   //replicationRecheckInterval is how often namenode checks for new replication work
   private long replicationRecheckInterval;
-  // default block size of a file
-  private long defaultBlockSize = 0;
+  private FsServerDefaults serverDefaults;
   // allow appending to hdfs files
   private boolean supportAppends = true;
 
@@ -413,7 +413,12 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
       10 * heartbeatInterval;
     this.replicationRecheckInterval = 
       conf.getInt("dfs.replication.interval", 3) * 1000L;
-    this.defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
+    this.serverDefaults = new FsServerDefaults(
+        conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE),
+        conf.getInt("io.bytes.per.checksum", DEFAULT_BYTES_PER_CHECKSUM),
+        conf.getInt("dfs.write.packet.size", DEFAULT_WRITE_PACKET_SIZE),
+        (short) conf.getInt("dfs.replication", DEFAULT_REPLICATION_FACTOR),
+        conf.getInt("io.file.buffer.size", DEFAULT_FILE_BUFFER_SIZE));
     this.maxFsObjects = conf.getLong("dfs.max.objects", 0);
     this.blockInvalidateLimit = Math.max(this.blockInvalidateLimit, 
                                          20*(int)(heartbeatInterval/1000));
@@ -505,7 +510,11 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
   }
 
   long getDefaultBlockSize() {
-    return defaultBlockSize;
+    return serverDefaults.getBlockSize();
+  }
+
+  FsServerDefaults getServerDefaults() {
+    return serverDefaults;
   }
 
   long getAccessTimePrecision() {

src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -545,6 +546,11 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
     return clientMachine;
   }
 
+  /** {@inheritDoc} */
+  public FsServerDefaults getServerDefaults() throws IOException {
+    return namesystem.getServerDefaults();
+  }
+
   /** {@inheritDoc} */
   public void create(String src, 
                      FsPermission masked,

src/test/hdfs/org/apache/hadoop/hdfs/TestDFSClientRetries.java

@@ -155,6 +155,8 @@ public class TestDFSClientRetries extends TestCase {
     
     public LocatedBlocks  getBlockLocations(String src, long offset, long length) throws IOException { return null; }
     
+    public FsServerDefaults getServerDefaults() throws IOException { return null; }
+    
     public void create(String src, FsPermission masked, String clientName, EnumSetWritable<CreateFlag> flag, short replication, long blockSize) throws IOException {}
     
     public LocatedBlock append(String src, String clientName) throws IOException { return null; }

src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.log4j.Level;
 
 
@@ -170,6 +172,33 @@ public class TestFileCreation extends junit.framework.TestCase {
     stm.close();
   }
 
+  /**
+   * Test that server default values can be retrieved on the client side
+   */
+  public void testServerDefaults() throws IOException {
+    Configuration conf = new Configuration();
+    conf.setLong("dfs.block.size", FSConstants.DEFAULT_BLOCK_SIZE);
+    conf.setInt("io.bytes.per.checksum", FSConstants.DEFAULT_BYTES_PER_CHECKSUM);
+    conf.setInt("dfs.write.packet.size", FSConstants.DEFAULT_WRITE_PACKET_SIZE);
+    conf.setInt("dfs.replication", FSConstants.DEFAULT_REPLICATION_FACTOR + 1);
+    conf.setInt("io.file.buffer.size", FSConstants.DEFAULT_FILE_BUFFER_SIZE);
+    MiniDFSCluster cluster = new MiniDFSCluster(conf,
+        FSConstants.DEFAULT_REPLICATION_FACTOR + 1, true, null);
+    cluster.waitActive();
+    FileSystem fs = cluster.getFileSystem();
+    try {
+      FsServerDefaults serverDefaults = fs.getServerDefaults();
+      assertEquals(FSConstants.DEFAULT_BLOCK_SIZE, serverDefaults.getBlockSize());
+      assertEquals(FSConstants.DEFAULT_BYTES_PER_CHECKSUM, serverDefaults.getBytesPerChecksum());
+      assertEquals(FSConstants.DEFAULT_WRITE_PACKET_SIZE, serverDefaults.getWritePacketSize());
+      assertEquals(FSConstants.DEFAULT_REPLICATION_FACTOR + 1, serverDefaults.getReplication());
+      assertEquals(FSConstants.DEFAULT_FILE_BUFFER_SIZE, serverDefaults.getFileBufferSize());
+    } finally {
+      fs.close();
+      cluster.shutdown();
+    }
+  }
+
   /**
    * Test that file data becomes available before file is closed.
    */