HDFS-7937. Erasure Coding: INodeFile quota computation unit tests. Contributed by Kai Sasaki.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java

@@ -203,8 +203,27 @@ public class BlockInfoStriped extends BlockInfo {
     // In case striped blocks, total usage by this striped blocks should
     // be the total of data blocks and parity blocks because
     // `getNumBytes` is the total of actual data block size.
-    return ((getNumBytes() - 1) / (dataBlockNum * BLOCK_STRIPED_CELL_SIZE) + 1)
-        * BLOCK_STRIPED_CELL_SIZE * parityBlockNum + getNumBytes();
+
+    // 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
+    long numBytesPerStripe = dataBlockNum * BLOCK_STRIPED_CELL_SIZE;
+    if (getNumBytes() % numBytesPerStripe == 0) {
+      return getNumBytes() / dataBlockNum * getTotalBlockNum();
+    }
+    // 1. Calculate the number of stripes in this block group. <Num Stripes>
+    long numStripes = (getNumBytes() - 1) / numBytesPerStripe + 1;
+    // 2. Calculate the parity cell length in the last stripe. Note that the
+    //    size of parity cells should equal the size of the first cell, if it
+    //    is not full. <Last Stripe Parity Cell Length>
+    long lastStripeParityCellLen = Math.min(getNumBytes() % numBytesPerStripe,
+        BLOCK_STRIPED_CELL_SIZE);
+    // 3. Total consumed space is the total of
+    //     - The total of the full cells of data blocks and parity blocks.
+    //     - The remaining of data block which does not make a stripe.
+    //     - The last parity block cells. These size should be same
+    //       to the first cell in this stripe.
+    return getTotalBlockNum() * (BLOCK_STRIPED_CELL_SIZE * (numStripes - 1))
+        + getNumBytes() % numBytesPerStripe
+        + lastStripeParityCellLen * parityBlockNum;
   }
 
   @Override

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java

@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import org.junit.Test;
+
+/**
+ * This class tests INodeFile with striped feature.
+ */
+public class TestStripedINodeFile {
+  public static final Log LOG = LogFactory.getLog(TestINodeFile.class);
+
+  private static final PermissionStatus perm = new PermissionStatus(
+      "userName", null, FsPermission.getDefault());
+
+  private static INodeFile createStripedINodeFile() {
+    return new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
+        null, (short)0, 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID);
+  }
+
+  @Test
+  public void testBlockStripedFeature()
+      throws IOException, InterruptedException{
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    assertTrue(inf.isStriped());
+  }
+
+  @Test
+  public void testBlockStripedTotalBlockCount() {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    Block blk = new Block(1);
+    BlockInfoStriped blockInfoStriped
+        = new BlockInfoStriped(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    assertEquals(9, blockInfoStriped.getTotalBlockNum());
+  }
+
+  @Test
+  public void testBlockStripedLength()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStriped blockInfoStriped
+        = new BlockInfoStriped(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    inf.addBlock(blockInfoStriped);
+    assertEquals(1, inf.getBlocks().length);
+  }
+
+  @Test
+  public void testBlockStripedConsumedSpace()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStriped blockInfoStriped
+        = new BlockInfoStriped(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    blockInfoStriped.setNumBytes(1);
+    inf.addBlock(blockInfoStriped);
+    //   0. Calculate the total bytes per stripes <Num Bytes per Stripes>
+    //   1. Calculate the number of stripes in this block group. <Num Stripes>
+    //   2. Calculate the last remaining length which does not make a stripe. <Last Stripe Length>
+    //   3. Total consumed space is the total of
+    //     a. The total of the full cells of data blocks and parity blocks.
+    //     b. The remaining of data block which does not make a stripe.
+    //     c. The last parity block cells. These size should be same
+    //        to the first cell in this stripe.
+    // So the total consumed space is the sum of
+    //  a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
+    //  b. <Num Bytes> % <Num Bytes per Stripes> = 1
+    //  c. <Last Stripe Length> * <Parity Block Num> = 1 * 3
+    assertEquals(4, inf.storagespaceConsumedWithStriped());
+    assertEquals(4, inf.storagespaceConsumed());
+  }
+
+  @Test
+  public void testMultipleBlockStripedConsumedSpace()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk1 = new Block(1);
+    BlockInfoStriped blockInfoStriped1
+        = new BlockInfoStriped(blk1,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    blockInfoStriped1.setNumBytes(1);
+    Block blk2 = new Block(2);
+    BlockInfoStriped blockInfoStriped2
+        = new BlockInfoStriped(blk2,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    blockInfoStriped2.setNumBytes(1);
+    inf.addBlock(blockInfoStriped1);
+    inf.addBlock(blockInfoStriped2);
+    // This is the double size of one block in above case.
+    assertEquals(4 * 2, inf.storagespaceConsumedWithStriped());
+    assertEquals(4 * 2, inf.storagespaceConsumed());
+  }
+
+  @Test
+  public void testBlockStripedFileSize()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStriped blockInfoStriped
+        = new BlockInfoStriped(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    blockInfoStriped.setNumBytes(100);
+    inf.addBlock(blockInfoStriped);
+    // Compute file size should return actual data
+    // size which is retained by this file.
+    assertEquals(100, inf.computeFileSize());
+    assertEquals(100, inf.computeFileSize(false, false));
+  }
+
+  @Test
+  public void testBlockStripedUCFileSize()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStripedUnderConstruction bInfoStripedUC
+        = new BlockInfoStripedUnderConstruction(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    bInfoStripedUC.setNumBytes(100);
+    inf.addBlock(bInfoStripedUC);
+    assertEquals(100, inf.computeFileSize());
+    assertEquals(0, inf.computeFileSize(false, false));
+  }
+
+  @Test
+  public void testBlockStripedComputeQuotaUsage()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStriped blockInfoStriped
+        = new BlockInfoStriped(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    blockInfoStriped.setNumBytes(100);
+    inf.addBlock(blockInfoStriped);
+
+    BlockStoragePolicySuite suite =
+        BlockStoragePolicySuite.createDefaultSuite();
+    QuotaCounts counts =
+        inf.computeQuotaUsageWithStriped(suite,
+            new QuotaCounts.Builder().build());
+    assertEquals(1, counts.getNameSpace());
+    // The total consumed space is the sum of
+    //  a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
+    //  b. <Num Bytes> % <Num Bytes per Stripes> = 100
+    //  c. <Last Stripe Length> * <Parity Block Num> = 100 * 3
+    assertEquals(400, counts.getStorageSpace());
+  }
+
+  @Test
+  public void testBlockStripedUCComputeQuotaUsage()
+      throws IOException, InterruptedException {
+    ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
+    INodeFile inf = createStripedINodeFile();
+    inf.addStripedBlocksFeature();
+    Block blk = new Block(1);
+    BlockInfoStripedUnderConstruction bInfoStripedUC
+        = new BlockInfoStripedUnderConstruction(blk,
+            (short)defaultSchema.getNumDataUnits(),
+            (short)defaultSchema.getNumParityUnits());
+    bInfoStripedUC.setNumBytes(100);
+    inf.addBlock(bInfoStripedUC);
+
+    BlockStoragePolicySuite suite
+        = BlockStoragePolicySuite.createDefaultSuite();
+    QuotaCounts counts
+        = inf.computeQuotaUsageWithStriped(suite,
+              new QuotaCounts.Builder().build());
+    assertEquals(1024, inf.getPreferredBlockSize());
+    assertEquals(1, counts.getNameSpace());
+    // Consumed space in the case of BlockInfoStripedUC can be calculated
+    // by using preferred block size. This is 1024 and total block num
+    // is 9(= 3 + 6). Consumed storage space should be 1024 * 9 = 9216.
+    assertEquals(9216, counts.getStorageSpace());
+  }
+}