HDFS-728. Create a comprehensive functional test for append. Contributed by Hairong Kuang.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/trunk@830003 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -318,6 +318,8 @@ Release 0.21.0 - Unreleased
     HDFS-730. Add 4 fault injection tests to simulate non-responsive datanode
     and out-of-memory problem for pipeline close ack.  (szetszwo)
     
+    HDFS-728. Create a comprehensive functional test for append. (hairong)
+
   BUG FIXES
 
     HDFS-76. Better error message to users when commands fail because of 

src/test/hdfs/org/apache/hadoop/hdfs/FileAppendTest4.java

@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/** This is a comprehensive append test that tries
+ * all combinations of file length and number of appended bytes
+ * In each iteration, it creates a file of len1. Then reopen
+ * the file for append. It first append len2 bytes, calls hflush,
+ * append len3 bytes and close the file. Afterwards, the content of
+ * the file is validated.
+ * Len1 ranges from [0, 2*BLOCK_SIZE+1], len2 ranges from [0, BLOCK_SIZE+1],
+ * and len3 ranges from [0, BLOCK_SIZE+1].
+ *
+ */
+public class FileAppendTest4 {
+  public static final Log LOG = LogFactory.getLog(FileAppendTest4.class);
+  
+  private static final int BYTES_PER_CHECKSUM = 4;
+  private static final int PACKET_SIZE = BYTES_PER_CHECKSUM;
+  private static final int BLOCK_SIZE = 2*PACKET_SIZE;
+  private static final short REPLICATION = 3;
+  private static final int DATANODE_NUM = 5;
+  private static Configuration conf;
+  private static MiniDFSCluster cluster;
+  private static DistributedFileSystem fs;
+
+  private static void init(Configuration conf) {
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY, PACKET_SIZE);
+    conf.setBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, true);
+  }
+  
+  @BeforeClass
+  public static void startUp () throws IOException {
+    conf = new HdfsConfiguration();
+    init(conf);
+    cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null);
+    fs = (DistributedFileSystem)cluster.getFileSystem();
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    cluster.shutdown();
+  }
+  
+  /**
+   * Comprehensive test for append 
+   * @throws IOException an exception might be thrown
+   */
+  @Test
+  public void testAppend() throws IOException {
+    final int maxOldFileLen = 2*BLOCK_SIZE+1;
+    final int maxFlushedBytes = BLOCK_SIZE;
+    byte[] contents = AppendTestUtil.initBuffer(
+        maxOldFileLen+2*maxFlushedBytes);
+    for (int oldFileLen =0; oldFileLen <=maxOldFileLen; oldFileLen++) {
+      for (int flushedBytes1=0; flushedBytes1<=maxFlushedBytes; 
+                                flushedBytes1++) {
+        for (int flushedBytes2=0; flushedBytes2 <=maxFlushedBytes; 
+                                  flushedBytes2++) {
+          final int fileLen = oldFileLen + flushedBytes1 + flushedBytes2;
+          // create the initial file of oldFileLen
+          final Path p = 
+            new Path("foo"+ oldFileLen +"_"+ flushedBytes1 +"_"+ flushedBytes2);
+          LOG.info("Creating file " + p);
+          FSDataOutputStream out = fs.create(p, false, 
+              conf.getInt("io.file.buffer.size", 4096), 
+              REPLICATION, BLOCK_SIZE);
+          out.write(contents, 0, oldFileLen);
+          out.close();
+
+          // append flushedBytes bytes to the file
+          out = fs.append(p);
+          out.write(contents, oldFileLen, flushedBytes1);
+          out.sync();
+
+          // write another flushedBytes2 bytes to the file
+          out.write(contents, oldFileLen + flushedBytes1, flushedBytes2);
+          out.close();
+
+          // validate the file content
+          AppendTestUtil.checkFullFile(fs, p, fileLen, contents, p.toString());
+          fs.delete(p, false);
+        }
+      }
+    }
+  }
+}