HDFS-730. Add 4 fault injection tests to simulate non-responsive datanode and out-of-memory problem for pipeline close ack.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/trunk@829887 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -312,6 +312,9 @@ Release 0.21.0 - Unreleased
     HDFS-616. Create functional tests for new design of the block report. (cos)
     
     HDFS-584. Fail the fault-inject build if any advices are mis-bound. (cos)
+
+    HDFS-730. Add 4 fault injection tests to simulate non-responsive datanode
+    and out-of-memory problem for pipeline close ack.  (szetszwo)
     
   BUG FIXES
 

src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.fi.FiTestUtil.Action;
 import org.apache.hadoop.fi.FiTestUtil.ActionContainer;
 import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
 import org.apache.hadoop.fi.FiTestUtil.CountdownConstraint;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -137,7 +138,8 @@ public class DataTransferTestUtil {
 
     /** {@inheritDoc} */
     public String toString() {
-      return currentTest + ", index=" + index;
+      return getClass().getSimpleName() + ":" + currentTest
+          + ", index=" + index;
     }
 
     /** return a String with this object and the datanodeID. */
@@ -147,6 +149,33 @@ public class DataTransferTestUtil {
     }
   }
 
+  /** An action to set a marker if the DatanodeID is matched. */
+  public static class DatanodeMarkingAction extends DataNodeAction {
+    private final MarkerConstraint marker;
+
+    /** Construct an object. */
+    public DatanodeMarkingAction(String currentTest, int index,
+        MarkerConstraint marker) {
+      super(currentTest, index);
+      this.marker = marker;
+    }
+
+    /** Set the marker if the DatanodeID is matched. */
+    @Override
+    public void run(DatanodeID datanodeid) throws IOException {
+      final DataTransferTest test = getDataTransferTest();
+      final Pipeline p = test.getPipeline(datanodeid);
+      if (p.contains(index, datanodeid)) {
+        marker.mark();
+      }
+    }
+
+    /** {@inheritDoc} */
+    public String toString() {
+      return super.toString() + ", " + marker;
+    }
+  }
+
   /** Throws OutOfMemoryError. */
   public static class OomAction extends DataNodeAction {
     /** Create an action for datanode i in the pipeline. */
@@ -273,8 +302,8 @@ public class DataTransferTestUtil {
     /** {@inheritDoc} */
     @Override
     public String toString() {
-      return super.toString()
-          + ", duration = [" + minDuration + "," + maxDuration + ")";
+      return super.toString() + ", duration="
+          + (maxDuration <= 0? "infinity": "[" + minDuration + ", " + maxDuration + ")");
     }
   }
 

src/test/aop/org/apache/hadoop/fi/FiTestUtil.java

@@ -174,4 +174,32 @@ public class FiTestUtil {
       action.run(parameter);
     }
   }
+
+  /** A MarkerConstraint is satisfied if it is marked. */
+  public static class MarkerConstraint implements Constraint {
+    private final String name;
+    private boolean marked = false;
+
+    /** Construct an object. */
+    public MarkerConstraint(String name) {
+      this.name = name;
+    }
+
+    /** Set marker to be marked. */
+    public void mark() {
+      marked = true;
+      LOG.info("Marking this " + this);
+    }
+
+    /** Is the marker marked? */
+    @Override
+    public boolean isSatisfied() {
+      return marked;
+    }
+
+    /** {@inheritDoc} */
+    public String toString() {
+      return getClass().getSimpleName() + "[" + name + ": " + marked + "]";
+    }
+  }
 }

src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj

@@ -47,11 +47,11 @@ public privileged aspect BlockReceiverAspects {
 	
   before(BlockReceiver blockreceiver
       ) throws IOException : callReceivePacket(blockreceiver) {
-    LOG.info("FI: callReceivePacket");
+    final DatanodeRegistration dr = blockreceiver.getDataNode().getDatanodeRegistration();
+    LOG.info("FI: callReceivePacket, datanode=" + dr.getName());
     DataTransferTest dtTest = DataTransferTestUtil.getDataTransferTest();
     if (dtTest != null)
-      dtTest.fiCallReceivePacket.run(
-          blockreceiver.getDataNode().getDatanodeRegistration());
+      dtTest.fiCallReceivePacket.run(dr);
 
     if (ProbabilityModel.injectCriteria(BlockReceiver.class.getSimpleName())) {
       LOG.info("Before the injection point");
@@ -73,7 +73,7 @@ public privileged aspect BlockReceiverAspects {
           lastPacketInBlock, len, endOfHeader) {
     if (len == 0) {
       final DatanodeRegistration dr = blockreceiver.getDataNode().getDatanodeRegistration();
-      LOG.info("FI: pipelineClose, datanode=" + dr
+      LOG.info("FI: pipelineClose, datanode=" + dr.getName()
           + ", offsetInBlock=" + offsetInBlock
           + ", seqno=" + seqno
           + ", lastPacketInBlock=" + lastPacketInBlock

src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java

@@ -22,13 +22,17 @@ import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fi.DataTransferTestUtil;
 import org.apache.hadoop.fi.FiTestUtil;
+import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
+import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.DoosAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
 import org.apache.hadoop.fi.DataTransferTestUtil.VerificationAction;
 import org.apache.hadoop.fi.FiTestUtil.Action;
+import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -313,6 +317,40 @@ public class TestFiDataTransferProtocol {
   private static void run41_43(String name, int i) throws IOException {
     runPipelineCloseTest(name, new SleepAction(name, i, 3000));
   }
+
+  private static void runPipelineCloseAck(String name, int i, DataNodeAction a
+      ) throws IOException {
+    FiTestUtil.LOG.info("Running " + name + " ...");
+    final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
+    final MarkerConstraint marker = new MarkerConstraint(name);
+    t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
+    t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID>(a, marker));
+    write1byte(name);
+  }
+
+  private static void run39_40(String name, int i) throws IOException {
+    runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
+  }
+
+  /**
+   * Pipeline close:
+   * DN1 never responses after received close ack DN2.
+   * Client gets an IOException and determine DN1 bad.
+   */
+  @Test
+  public void pipeline_Fi_39() throws IOException {
+    run39_40(FiTestUtil.getMethodName(), 1);
+  }
+
+  /**
+   * Pipeline close:
+   * DN0 never responses after received close ack DN1.
+   * Client gets an IOException and determine DN0 bad.
+   */
+  @Test
+  public void pipeline_Fi_40() throws IOException {
+    run39_40(FiTestUtil.getMethodName(), 0);
+  }
   
   /**
    * Pipeline close with DN0 very slow but it won't lead to timeout.
@@ -377,6 +415,32 @@ public class TestFiDataTransferProtocol {
     runPipelineCloseTest(methodName, new OomAction(methodName, 2));
   }
 
+  private static void run47_48(String name, int i) throws IOException {
+    runPipelineCloseAck(name, i, new OomAction(name, i));
+  }
+
+  /**
+   * Pipeline close:
+   * DN1 throws an OutOfMemoryException right after
+   * it received a close ack from DN2.
+   * Client gets an IOException and determine DN1 bad.
+   */
+  @Test
+  public void pipeline_Fi_47() throws IOException {
+    run47_48(FiTestUtil.getMethodName(), 1);
+  }
+
+  /**
+   * Pipeline close:
+   * DN0 throws an OutOfMemoryException right after
+   * it received a close ack from DN1.
+   * Client gets an IOException and determine DN0 bad.
+   */
+  @Test
+  public void pipeline_Fi_48() throws IOException {
+    run47_48(FiTestUtil.getMethodName(), 0);
+  }
+
   private static void runBlockFileCloseTest(String methodName,
       Action<DatanodeID> a) throws IOException {
     FiTestUtil.LOG.info("Running " + methodName + " ...");