فهرست منبع

HDFS-15039. Cache meta file length of FinalizedReplica to reduce call File.length(). Contributed by Yang Yun.

Wei-Chiu Chuang 5 سال پیش
والد
کامیت
20903f72b4

+ 9 - 0
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
  */
 public class FinalizedReplica extends LocalReplica {
   private byte[] lastPartialChunkChecksum;
+  private int metaLength = -1;
   /**
    * Constructor.
    * @param blockId block id
@@ -144,6 +145,14 @@ public class FinalizedReplica extends LocalReplica {
         " does not support createInfo");
   }
 
+  @Override
+  public long getMetadataLength() {
+    if (metaLength < 0) {
+      metaLength = (int)super.getMetadataLength();
+    }
+    return metaLength;
+  }
+
   public byte[] getLastPartialChunkChecksum() {
     return lastPartialChunkChecksum;
   }

+ 36 - 0
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 import com.google.common.base.Supplier;
 import com.google.common.collect.Lists;
 
+import java.io.FileInputStream;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import org.apache.commons.io.FileUtils;
@@ -987,4 +989,38 @@ public class TestFsDatasetImpl {
       assertTrue(f.exists());
     }
   }
+
+  @Test
+  public void testGetMetadataLengthOfFinalizedReplica() throws IOException {
+    FsVolumeImpl fsv1 = Mockito.mock(FsVolumeImpl.class);
+    File blockDir = new File(BASE_DIR,"testFinalizedReplica/block");
+    if (!blockDir.exists()) {
+      assertTrue(blockDir.mkdirs());
+    }
+    long blockID = 1;
+    long genStamp = 2;
+    File metaFile = new File(blockDir,Block.BLOCK_FILE_PREFIX +
+        blockID + "_" + genStamp + Block.METADATA_EXTENSION);
+
+    // create meta file on disk
+    OutputStream os = new FileOutputStream(metaFile);
+    os.write("TEST_META_SIZE".getBytes());
+    os.close();
+    long fileLength = metaFile.length();
+
+    ReplicaInfo replica = new FinalizedReplica(
+        blockID, 2, genStamp, fsv1, blockDir);
+
+    long metaLength = replica.getMetadataLength();
+    assertEquals(fileLength, metaLength);
+
+    // Delete the meta file on disks, make sure we still can get the length
+    // from cached meta size.
+    metaFile.delete();
+    metaLength = replica.getMetadataLength();
+    assertEquals(fileLength, metaLength);
+    if (!blockDir.exists()) {
+      assertTrue(blockDir.delete());
+    }
+  }
 }