HADOOP-3177. Implement Syncable interface for FileSystem.

(Tsz Wo (Nicholas), SZE via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@663326 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -126,6 +126,9 @@ Trunk (unreleased changes)
     HADOOP-3250. Extend FileSystem API to allow appending to files.
     (Tsz Wo (Nicholas), SZE via cdouglas)
 
+    HADOOP-3177. Implement Syncable interface for FileSystem.
+    (Tsz Wo (Nicholas), SZE via dhruba)
+
   IMPROVEMENTS
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().

src/java/org/apache/hadoop/dfs/DFSClient.java

@@ -1699,7 +1699,7 @@ class DFSClient implements FSConstants {
    * datanode from the original pipeline. The DataStreamer now
    * starts sending packets from the dataQueue.
   ****************************************************************/
-  class DFSOutputStream extends FSOutputSummer {
+  class DFSOutputStream extends FSOutputSummer implements Syncable {
     private Socket s;
     boolean closed = false;
   
@@ -2497,7 +2497,7 @@ class DFSClient implements FSConstants {
      * that data has been flushed to persistent store on the 
      * datanode. Block allocations are persisted on namenode.
      */
-    public synchronized void fsync() throws IOException {
+    public synchronized void sync() throws IOException {
       try {
         /* Record current blockOffset. This might be changed inside
          * flushBuffer() where a partial checksum chunk might be flushed.

src/java/org/apache/hadoop/fs/FSDataOutputStream.java

@@ -22,7 +22,7 @@ import java.io.*;
 /** Utility that wraps a {@link OutputStream} in a {@link DataOutputStream},
  * buffers output through a {@link BufferedOutputStream} and creates a checksum
  * file. */
-public class FSDataOutputStream extends DataOutputStream {
+public class FSDataOutputStream extends DataOutputStream implements Syncable {
   private OutputStream wrappedStream;
 
   private static class PositionCache extends FilterOutputStream {
@@ -83,4 +83,11 @@ public class FSDataOutputStream extends DataOutputStream {
   public OutputStream getWrappedStream() {
     return wrappedStream;
   }
+
+  /** {@inheritDoc} */
+  public void sync() throws IOException {
+    if (wrappedStream instanceof Syncable) {
+      ((Syncable)wrappedStream).sync();
+    }
+  }
 }

src/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -180,7 +180,7 @@ public class RawLocalFileSystem extends FileSystem {
   /*********************************************************
    * For create()'s FSOutputStream.
    *********************************************************/
-  class LocalFSFileOutputStream extends OutputStream {
+  class LocalFSFileOutputStream extends OutputStream implements Syncable {
     FileOutputStream fos;
     
     private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
@@ -207,6 +207,11 @@ public class RawLocalFileSystem extends FileSystem {
         throw new FSError(e);                // assume native fs error
       }
     }
+
+    /** {@inheritDoc} */
+    public void sync() throws IOException {
+      fos.getFD().sync();      
+    }
   }
   
   /** {@inheritDoc} */

src/java/org/apache/hadoop/fs/Syncable.java

@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/** This interface declare the sync() operation. */
+public interface Syncable {
+  /**
+   * Synchronize all buffer with the underlying devices.
+   * @throws IOException
+   */
+  public void sync() throws IOException;
+}

src/test/org/apache/hadoop/dfs/TestAbandonBlock.java

@@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.net.NetUtils;
 
 public class TestAbandonBlock extends junit.framework.TestCase {
   public static final Log LOG = LogFactory.getLog(TestAbandonBlock.class);
@@ -33,12 +32,6 @@ public class TestAbandonBlock extends junit.framework.TestCase {
   static final String FILE_NAME_PREFIX
       = "/" + TestAbandonBlock.class.getSimpleName() + "_"; 
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   public void testAbandonBlock() throws IOException {
     MiniDFSCluster cluster = new MiniDFSCluster(CONF, 2, true, null);
     FileSystem fs = cluster.getFileSystem();
@@ -51,7 +44,7 @@ public class TestAbandonBlock extends junit.framework.TestCase {
       for(int i = 0; i < 1024; i++) {
         fout.write(123);
       }
-      flushFile(fout);
+      fout.sync();
   
       //try reading the block by someone
       DFSClient dfsclient = new DFSClient(CONF);

src/test/org/apache/hadoop/dfs/TestFileAppend.java

@@ -63,12 +63,6 @@ public class TestFileAppend extends TestCase {
     return stm;
   }
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   //
   // writes to file but does not close it
   //
@@ -234,14 +228,14 @@ public class TestFileAppend extends TestCase {
       // write to file
       int mid = fileSize/2;
       stm.write(fileContents, 0, mid);
-      flushFile(stm);
+      stm.sync();
       System.out.println("Wrote and Flushed first part of file.");
 
       // write the remainder of the file
       stm.write(fileContents, mid, fileSize - mid);
       System.out.println("Written second part of file");
-      flushFile(stm);
-      flushFile(stm); // two consecutive flushes is being tested here.
+      stm.sync();
+      stm.sync();
       System.out.println("Wrote and Flushed second part of file.");
 
       // verify that full blocks are sane
@@ -287,7 +281,7 @@ public class TestFileAppend extends TestCase {
       int start = 0;
       for (start = 0; (start + 29) < fileSize; ) {
         stm.write(fileContents, start, 29);
-        flushFile(stm);
+        stm.sync();
         start += 29;
       }
       stm.write(fileContents, start, fileSize-start);

src/test/org/apache/hadoop/dfs/TestFileCreation.java

@@ -62,12 +62,6 @@ public class TestFileCreation extends junit.framework.TestCase {
     return stm;
   }
 
-  private void flushFile(FSDataOutputStream stm) throws IOException {
-    DFSClient.DFSOutputStream dfstream = (DFSClient.DFSOutputStream)
-                                            (stm.getWrappedStream());
-    dfstream.fsync();
-  }
-
   //
   // writes to file but does not close it
   //
@@ -439,7 +433,7 @@ public class TestFileCreation extends junit.framework.TestCase {
 
       // write two full blocks.
       writeFile(stm, numBlocks * blockSize);
-      flushFile(stm);
+      stm.sync();
 
       // rename file wile keeping it open.
       Path fileRenamed = new Path("/filestatusRenamed.dat");
@@ -657,7 +651,7 @@ public class TestFileCreation extends junit.framework.TestCase {
       final Path fpath = new Path(f);
       FSDataOutputStream out = TestFileCreation.createFile(dfs, fpath, DATANODE_NUM);
       out.write("something".getBytes());
-      ((DFSClient.DFSOutputStream)out.getWrappedStream()).fsync();
+      out.sync();
 
       // set the soft and hard limit to be 1 second so that the
       // namenode triggers lease recovery