HADOOP-8365. Add flag to disable durable sync. Contributed by Eli Collins

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1360011 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -181,6 +181,8 @@ Release 1.1.0 - unreleased
 
     HDFS-3516. Check content-type in WebHdfsFileSystem.  (szetszwo)
 
+    HADOOP-8365. Provide ability to disable working sync. (eli)
+
   BUG FIXES
 
     MAPREDUCE-4087. [Gridmix] GenerateDistCacheData job of Gridmix can

src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -228,6 +228,7 @@ public class DataNode extends Configured
   int socketWriteTimeout = 0;  
   boolean transferToAllowed = true;
   int writePacketSize = 0;
+  private boolean durableSync;
   boolean isBlockTokenEnabled;
   BlockTokenSecretManager blockTokenSecretManager;
   boolean isBlockTokenInitialized = false;
@@ -294,6 +295,7 @@ public class DataNode extends Configured
         DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
 
     datanodeObject = this;
+    durableSync = conf.getBoolean("dfs.durable.sync", true);
     this.userWithLocalPathAccess = conf
         .get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
     try {
@@ -755,11 +757,12 @@ public class DataNode extends Configured
       dnRegistration.exportedKeys = ExportedBlockKeys.DUMMY_KEYS;
     }
 
-
-    Block[] bbwReport = data.getBlocksBeingWrittenReport();
-    long[] blocksBeingWritten =
-      BlockListAsLongs.convertToArrayLongs(bbwReport);
-    namenode.blocksBeingWrittenReport(dnRegistration, blocksBeingWritten);
+    if (durableSync) {
+      Block[] bbwReport = data.getBlocksBeingWrittenReport();
+      long[] blocksBeingWritten =
+        BlockListAsLongs.convertToArrayLongs(bbwReport);
+      namenode.blocksBeingWrittenReport(dnRegistration, blocksBeingWritten);
+    }
 
     // random short delay - helps scatter the BR from all DNs
     // - but we can start generating the block report immediately

src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java

@@ -347,6 +347,7 @@ public class FSDataset implements FSConstants, FSDatasetInterface {
       this.reserved = conf.getLong("dfs.datanode.du.reserved", 0);
       this.dataDir = new FSDir(currentDir);
       this.currentDir = currentDir;
+      boolean durableSync = conf.getBoolean("dfs.durable.sync", true);
       File parent = currentDir.getParentFile();
 
       this.detachDir = new File(parent, "detach");
@@ -366,7 +367,11 @@ public class FSDataset implements FSConstants, FSDatasetInterface {
       // should not be deleted.
       blocksBeingWritten = new File(parent, "blocksBeingWritten");
       if (blocksBeingWritten.exists()) {
-        recoverBlocksBeingWritten(blocksBeingWritten);
+        if (durableSync) {  
+          recoverBlocksBeingWritten(blocksBeingWritten);
+        } else {
+          FileUtil.fullyDelete(blocksBeingWritten);
+        }
       }
       
       if (!blocksBeingWritten.mkdirs()) {

src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -316,6 +316,8 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
   private long defaultBlockSize = 0;
   // allow file appending (for test coverage)
   private boolean allowBrokenAppend = false;
+  // enable durable sync
+  private boolean durableSync = true;
 
   /**
    * Last block index used for replication work.
@@ -538,6 +540,11 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
                "however append is not supported. This configuration option " +
                "is no longer required to enable sync.");
     }
+    this.durableSync = conf.getBoolean("dfs.durable.sync", true);
+    if (!durableSync) {
+      LOG.warn("Durable sync disabled. Beware data loss when running " +
+               "programs like HBase that require durable sync!");
+    }
     this.isAccessTokenEnabled = conf.getBoolean(
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, false);
     if (isAccessTokenEnabled) {
@@ -2312,11 +2319,13 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean,
     }
 
     // If this commit does not want to close the file, persist
-    // blocks and return
+    // blocks (if durable sync is enabled) and return
     src = leaseManager.findPath(pendingFile);
     if (!closeFile) {
-      dir.persistBlocks(src, pendingFile);
-      getEditLog().logSync();
+      if (durableSync) {
+        dir.persistBlocks(src, pendingFile);
+        getEditLog().logSync();
+      }
       LOG.info("commitBlockSynchronization(" + lastblock + ") successful");
       return;
     }