HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance issues in testing. Contributed by Rui Li.

Change-Id: I9856456b59ed881c5ba2acce51e4d9bd01dc6f48

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -613,6 +613,9 @@ Trunk (Unreleased)
       HADOOP-12047. Indicate preference not to affect input buffers during
       coding in erasure coder. (Kai Zheng via waltersu4549)
 
+      HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance
+      issues in testing. (Rui Li via zhz)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawDecoder.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A dummy raw decoder that does no real computation.
+ * Instead, it just returns zero bytes.
+ * This decoder can be used to isolate the performance issue to HDFS side logic
+ * instead of codec, and is intended for test only.
+ */
+@InterfaceAudience.Private
+public class DummyRawDecoder extends AbstractRawErasureDecoder {
+  public DummyRawDecoder(int numDataUnits, int numParityUnits) {
+    super(numDataUnits, numParityUnits);
+  }
+
+  @Override
+  protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+      ByteBuffer[] outputs) {
+    // Nothing to do. Output buffers have already been reset
+  }
+
+  @Override
+  protected void doDecode(byte[][] inputs, int[] inputOffsets, int dataLen,
+      int[] erasedIndexes, byte[][] outputs, int[] outputOffsets) {
+    // Nothing to do. Output buffers have already been reset
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawEncoder.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A dummy raw encoder that does no real computation.
+ * Instead, it just returns zero bytes.
+ * This encoder can be used to isolate the performance issue to HDFS side logic
+ * instead of codec, and is intended for test only.
+ */
+@InterfaceAudience.Private
+public class DummyRawEncoder extends AbstractRawErasureEncoder {
+  public DummyRawEncoder(int numDataUnits, int numParityUnits) {
+    super(numDataUnits, numParityUnits);
+  }
+
+  @Override
+  protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+    // Nothing to do. Output buffers have already been reset
+  }
+
+  @Override
+  protected void doEncode(byte[][] inputs, int[] inputOffsets, int dataLen,
+      byte[][] outputs, int[] outputOffsets) {
+    // Nothing to do. Output buffers have already been reset
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java

@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * A raw erasure coder factory for dummy raw coders.
+ */
+@InterfaceAudience.Private
+public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
+  @Override
+  public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
+    return new DummyRawEncoder(numDataUnits, numParityUnits);
+  }
+
+  @Override
+  public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
+    return new DummyRawDecoder(numDataUnits, numParityUnits);
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java

@@ -75,6 +75,10 @@ public abstract class TestCoderBase {
     this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default
   }
 
+  protected byte[] getZeroChunkBytes() {
+    return zeroChunkBytes;
+  }
+
   protected void prepareBufferAllocator(boolean usingSlicedBuffer) {
     if (usingSlicedBuffer) {
       int roughEstimationSpace =

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java

@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Test dummy raw coder.
+ */
+public class TestDummyRawCoder extends TestRawCoderBase {
+  @Before
+  public void setup() {
+    encoderClass = DummyRawEncoder.class;
+    decoderClass = DummyRawDecoder.class;
+    setAllowDump(false);
+    setChunkSize(baseChunkSize);
+  }
+
+  @Test
+  public void testCoding_6x3_erasing_d0_d2() {
+    prepare(null, 6, 3, new int[]{0, 2}, new int[0], false);
+    testCodingDoMixed();
+  }
+
+  @Test
+  public void testCoding_6x3_erasing_d0_p0() {
+    prepare(null, 6, 3, new int[]{0}, new int[]{0}, false);
+    testCodingDoMixed();
+  }
+
+  @Override
+  protected void testCoding(boolean usingDirectBuffer) {
+    this.usingDirectBuffer = usingDirectBuffer;
+    prepareCoders();
+
+    prepareBufferAllocator(true);
+    setAllowChangeInputs(false);
+
+    // Generate data and encode
+    ECChunk[] dataChunks = prepareDataChunksForEncoding();
+    markChunks(dataChunks);
+    ECChunk[] parityChunks = prepareParityChunksForEncoding();
+    encoder.encode(dataChunks, parityChunks);
+    compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length));
+
+    // Decode
+    restoreChunksFromMark(dataChunks);
+    backupAndEraseChunks(dataChunks, parityChunks);
+    ECChunk[] inputChunks = prepareInputChunksForDecoding(
+        dataChunks, parityChunks);
+    ensureOnlyLeastRequiredChunks(inputChunks);
+    ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
+    decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+    compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length));
+  }
+
+  private ECChunk[] getEmptyChunks(int num) {
+    ECChunk[] chunks = new ECChunk[num];
+    for (int i = 0; i < chunks.length; i++) {
+      chunks[i] = new ECChunk(ByteBuffer.wrap(getZeroChunkBytes()));
+    }
+    return chunks;
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java

@@ -30,8 +30,8 @@ import java.lang.reflect.Constructor;
 public abstract class TestRawCoderBase extends TestCoderBase {
   protected Class<? extends RawErasureEncoder> encoderClass;
   protected Class<? extends RawErasureDecoder> decoderClass;
-  private RawErasureEncoder encoder;
-  private RawErasureDecoder decoder;
+  protected RawErasureEncoder encoder;
+  protected RawErasureDecoder decoder;
 
   /**
    * Doing twice to test if the coders can be repeatedly reused. This matters
@@ -187,13 +187,13 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     compareAndVerify(backupChunks, recoveredChunks);
   }
 
-  private void setAllowChangeInputs(boolean allowChangeInputs) {
+  protected void setAllowChangeInputs(boolean allowChangeInputs) {
     this.allowChangeInputs = allowChangeInputs;
     encoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
     decoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
   }
 
-  private void prepareCoders() {
+  protected void prepareCoders() {
     if (encoder == null) {
       encoder = createEncoder();
     }
@@ -203,7 +203,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     }
   }
 
-  private void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
+  protected void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
     int leastRequiredNum = numDataUnits;
     int erasedNum = erasedDataIndexes.length + erasedParityIndexes.length;
     int goodNum = inputChunks.length - erasedNum;