HDFS-1024 SecondaryNamenode fails to checkpoint because namenode fails with CancelledKeyException (Dmytro Molkov via Stack)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20@931031 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -14,6 +14,9 @@ Release 0.20.3 - Unreleased
     MAPREDUCE-1522. FileInputFormat may use the default FileSystem for the
     input path. (Tsz Wo (Nicholas), SZE via cdouglas)
 
+    HDFS-1024 SecondaryNamenode fails to checkpoint because namenode fails
+    with CancelledKeyException (Dmytro Molkov via Stack)
+
   IMPROVEMENTS
 
     MAPREDUCE-1407. Update javadoc in mapreduce.{Mapper,Reducer} to match

src/hdfs/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java

@@ -45,10 +45,14 @@ public class GetImageServlet extends HttpServlet {
       FSImage nnImage = (FSImage)context.getAttribute("name.system.image");
       TransferFsImage ff = new TransferFsImage(pmap, request, response);
       if (ff.getImage()) {
+        response.setHeader(TransferFsImage.CONTENT_LENGTH,
+          String.valueOf(nnImage.getFsImageName().length()));
         // send fsImage
         TransferFsImage.getFileServer(response.getOutputStream(),
                                       nnImage.getFsImageName()); 
       } else if (ff.getEdit()) {
+        response.setHeader(TransferFsImage.CONTENT_LENGTH,
+          String.valueOf(nnImage.getFsEditName().length()));
         // send edits
         TransferFsImage.getFileServer(response.getOutputStream(),
                                       nnImage.getFsEditName());

src/hdfs/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java

@@ -21,6 +21,7 @@ import java.io.*;
 import java.net.*;
 import java.util.Iterator;
 import java.util.Map;
+import java.lang.Math;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpServletRequest;
 
@@ -32,6 +33,8 @@ import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.ErrorSimulator;
  */
 class TransferFsImage implements FSConstants {
   
+  public final static String CONTENT_LENGTH = "Content-Length";
+  
   private boolean isGetImage;
   private boolean isGetEdit;
   private boolean isPutImage;
@@ -118,6 +121,16 @@ class TransferFsImage implements FSConstants {
         throw new IOException("If this exception is not caught by the " +
             "name-node fs image will be truncated.");
       }
+      
+      if (ErrorSimulator.getErrorSimulation(3)
+          && localfile.getAbsolutePath().contains("fsimage")) {
+          // Test sending image shorter than localfile
+          long len = localfile.length();
+          buf = new byte[(int)Math.min(len/2, BUFFER_SIZE)];
+          // This will read at most half of the image
+          // and the rest of the image will be sent over the wire
+          infile.read(buf);
+      }
       int num = 1;
       while (num > 0) {
         num = infile.read(buf);
@@ -148,6 +161,15 @@ class TransferFsImage implements FSConstants {
     //
     URL url = new URL(str.toString());
     URLConnection connection = url.openConnection();
+    long advertisedSize;
+    String contentLength = connection.getHeaderField(CONTENT_LENGTH);
+    if (contentLength != null) {
+      advertisedSize = Long.parseLong(contentLength);
+    } else {
+      throw new IOException(CONTENT_LENGTH + " header is not provided " +
+                            "by the namenode when trying to fetch " + str);
+    }
+    long received = 0;
     InputStream stream = connection.getInputStream();
     FileOutputStream[] output = null;
 
@@ -162,6 +184,7 @@ class TransferFsImage implements FSConstants {
       while (num > 0) {
         num = stream.read(buf);
         if (num > 0 && localPath != null) {
+          received += num;
           for (int i = 0; i < output.length; i++) {
             output[i].write(buf, 0, num);
           }
@@ -176,6 +199,11 @@ class TransferFsImage implements FSConstants {
           }
         }
       }
+      if (received != advertisedSize) {
+        throw new IOException("File " + str + " received length " + received +
+                              " is not of the advertised size " +
+                              advertisedSize);
+      }
     }
   }
 }

src/test/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java

@@ -382,6 +382,55 @@ public class TestCheckpoint extends TestCase {
     }
   }
 
+  /**
+   * Simulate namenode failing to send the whole file
+   * secondary namenode sometimes assumed it received all of it
+   */
+  @SuppressWarnings("deprecation")
+  void testNameNodeImageSendFail(Configuration conf)
+    throws IOException {
+    System.out.println("Starting testNameNodeImageSendFail");
+    Path file1 = new Path("checkpointww.dat");
+    MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, 
+                                                false, null);
+    cluster.waitActive();
+    FileSystem fileSys = cluster.getFileSystem();
+    try {
+      assertTrue(!fileSys.exists(file1));
+      //
+      // Make the checkpoint fail after rolling the edit log.
+      //
+      SecondaryNameNode secondary = startSecondaryNameNode(conf);
+      ErrorSimulator.setErrorSimulation(3);
+
+      try {
+        secondary.doCheckpoint();  // this should fail
+        fail("Did not get expected exception");
+      } catch (IOException e) {
+        // We only sent part of the image. Have to trigger this exception
+        assertTrue(e.getMessage().contains("is not of the advertised size"));
+      }
+      ErrorSimulator.clearErrorSimulation(3);
+      secondary.shutdown(); // secondary namenode crash!
+
+      // start new instance of secondary and verify that 
+      // a new rollEditLog suceedes inspite of the fact that 
+      // edits.new already exists.
+      //
+      secondary = startSecondaryNameNode(conf);
+      secondary.doCheckpoint();  // this should work correctly
+      secondary.shutdown();
+
+      //
+      // Create a new file
+      //
+      writeFile(fileSys, file1, replication);
+      checkFile(fileSys, file1, replication);
+    } finally {
+      fileSys.close();
+      cluster.shutdown();
+    }
+  }
   /**
    * Test different startup scenarios.
    * <p><ol>
@@ -592,7 +641,7 @@ public class TestCheckpoint extends TestCase {
       // Take a checkpoint
       //
       SecondaryNameNode secondary = startSecondaryNameNode(conf);
-      ErrorSimulator.initializeErrorSimulationEvent(3);
+      ErrorSimulator.initializeErrorSimulationEvent(4);
       secondary.doCheckpoint();
       secondary.shutdown();
     } finally {
@@ -646,6 +695,7 @@ public class TestCheckpoint extends TestCase {
 
     // file2 is left behind.
 
+    testNameNodeImageSendFail(conf);
     testSecondaryNamenodeError1(conf);
     testSecondaryNamenodeError2(conf);
     testSecondaryNamenodeError3(conf);