HDFS-110268. Correctly reconstruct erasure coding file from FSImage. Contributed by SammiChen.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java

@@ -332,7 +332,9 @@ public final class FSImageFormatPBINode {
       short replication = (short) f.getReplication();
       BlockType blockType = PBHelperClient.convert(f.getBlockType());
       LoaderContext state = parent.getLoaderContext();
-      ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
+      ErasureCodingPolicy ecPolicy = (blockType == BlockType.STRIPED) ?
+          ErasureCodingPolicyManager.getPolicyByPolicyID((byte) replication) :
+          null;
 
       BlockInfo[] blocks = new BlockInfo[bp.size()];
       for (int i = 0; i < bp.size(); ++i) {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java

@@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -31,6 +33,7 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.EnumSet;
 
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
@@ -73,8 +76,9 @@ public class TestFSImage {
 
   private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
       "image-with-zero-block-size.tar.gz";
-  private static final ErasureCodingPolicy testECPolicy
-      = ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private static final ErasureCodingPolicy testECPolicy =
+      ErasureCodingPolicyManager.getPolicyByPolicyID(
+          HdfsConstants.RS_10_4_POLICY_ID);
 
   @Test
   public void testPersist() throws IOException {
@@ -446,8 +450,8 @@ public class TestFSImage {
   /**
    * Ensure that FSImage supports BlockGroup.
    */
-  @Test
-  public void testSupportBlockGroup() throws IOException {
+  @Test(timeout = 60000)
+  public void testSupportBlockGroup() throws Exception {
     final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() +
         testECPolicy.getNumParityUnits());
     final int BLOCK_SIZE = 8 * 1024 * 1024;
@@ -459,32 +463,82 @@ public class TestFSImage {
           .build();
       cluster.waitActive();
       DistributedFileSystem fs = cluster.getFileSystem();
-      fs.getClient().getNamenode().setErasureCodingPolicy("/", testECPolicy);
-      Path file = new Path("/striped");
-      FSDataOutputStream out = fs.create(file);
-      byte[] bytes = DFSTestUtil.generateSequentialBytes(0, BLOCK_SIZE);
-      out.write(bytes);
-      out.close();
-
+      Path parentDir = new Path("/ec-10-4");
+      Path childDir = new Path(parentDir, "ec-3-2");
+      ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager
+          .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID);
+
+      // Create directories and files
+      fs.mkdirs(parentDir);
+      fs.mkdirs(childDir);
+      fs.setErasureCodingPolicy(parentDir, testECPolicy);
+      fs.setErasureCodingPolicy(childDir, ec32Policy);
+      Path file_10_4 = new Path(parentDir, "striped_file_10_4");
+      Path file_3_2 = new Path(childDir, "striped_file_3_2");
+
+      // Write content to files
+      byte[] bytes = StripedFileTestUtil.generateBytes(BLOCK_SIZE);
+      DFSTestUtil.writeFile(fs, file_10_4, new String(bytes));
+      DFSTestUtil.writeFile(fs, file_3_2, new String(bytes));
+
+      // Save namespace and restart NameNode
       fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
       fs.saveNamespace();
       fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
 
       cluster.restartNameNodes();
       fs = cluster.getFileSystem();
-      assertTrue(fs.exists(file));
+      assertTrue(fs.exists(file_10_4));
+      assertTrue(fs.exists(file_3_2));
 
-      // check the information of striped blocks
+      // check the information of file_10_4
       FSNamesystem fsn = cluster.getNamesystem();
-      INodeFile inode = fsn.dir.getINode(file.toString()).asFile();
+      INodeFile inode = fsn.dir.getINode(file_10_4.toString()).asFile();
       assertTrue(inode.isStriped());
       BlockInfo[] blks = inode.getBlocks();
       assertEquals(1, blks.length);
       assertTrue(blks[0].isStriped());
+      assertEquals(testECPolicy.getId(),
+          fs.getErasureCodingPolicy(file_10_4).getId());
+      assertEquals(testECPolicy.getId(),
+          ((BlockInfoStriped)blks[0]).getErasureCodingPolicy().getId());
       assertEquals(testECPolicy.getNumDataUnits(),
           ((BlockInfoStriped) blks[0]).getDataBlockNum());
       assertEquals(testECPolicy.getNumParityUnits(),
           ((BlockInfoStriped) blks[0]).getParityBlockNum());
+      byte[] content = DFSTestUtil.readFileAsBytes(fs, file_10_4);
+      assertArrayEquals(bytes, content);
+
+
+      // check the information of file_3_2
+      inode = fsn.dir.getINode(file_3_2.toString()).asFile();
+      assertTrue(inode.isStriped());
+      blks = inode.getBlocks();
+      assertEquals(1, blks.length);
+      assertTrue(blks[0].isStriped());
+      assertEquals(ec32Policy.getId(),
+          fs.getErasureCodingPolicy(file_3_2).getId());
+      assertEquals(ec32Policy.getNumDataUnits(),
+          ((BlockInfoStriped) blks[0]).getDataBlockNum());
+      assertEquals(ec32Policy.getNumParityUnits(),
+          ((BlockInfoStriped) blks[0]).getParityBlockNum());
+      content = DFSTestUtil.readFileAsBytes(fs, file_3_2);
+      assertArrayEquals(bytes, content);
+
+      // check the EC policy on parent Dir
+      ErasureCodingPolicy ecPolicy =
+          fsn.getErasureCodingPolicy(parentDir.toString());
+      assertNotNull(ecPolicy);
+      assertEquals(testECPolicy.getId(), ecPolicy.getId());
+
+      // check the EC policy on child Dir
+      ecPolicy = fsn.getErasureCodingPolicy(childDir.toString());
+      assertNotNull(ecPolicy);
+      assertEquals(ec32Policy.getId(), ecPolicy.getId());
+
+      // check the EC policy on root directory
+      ecPolicy = fsn.getErasureCodingPolicy("/");
+      assertNull(ecPolicy);
     } finally {
       if (cluster != null) {
         cluster.shutdown();