HDFS-436. Introduce AspectJ framework for HDFS code and tests. Contributed by Konstantin Boudnik

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/trunk@787544 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -2,6 +2,11 @@ Hadoop HDFS Change Log
 
 Trunk (unreleased changes)
 
+  NEW FEATURES
+
+    HDFS-436. Introduce AspectJ framework for HDFS code and tests.
+    (Konstantin Boudnik via szetszwo)
+
   IMPROVEMENTS
 
     HDFS-381. Remove blocks from DataNode maps when corresponding file

build.xml

@@ -315,6 +315,31 @@
     </copy>
   </target>
 
+  <!-- Weaving aspects in place 
+  	Later on one can run 'ant jar' to create Hadoop jar file with instrumented classes
+  -->
+  <target name="injectfaults" depends="compile" description="Weaves aspects into precomplied HDFS classes">
+    <!-- AspectJ task definition -->
+    <taskdef resource="org/aspectj/tools/ant/taskdefs/aspectjTaskdefs.properties">
+      <classpath>
+        <pathelement location="${common.ivy.lib.dir}/aspectjtools-1.6.4.jar"/>
+      </classpath>
+    </taskdef>
+    <echo message="Start weaving aspects in place"/>
+    <iajc
+      encoding="${build.encoding}" 
+      srcdir="${hdfs.src.dir};${build.src};${test.src.dir}/aop" 
+      includes="org/apache/hadoop/**/*.java, org/apache/hadoop/**/*.aj"
+      destDir="${build.classes}"
+      debug="${javac.debug}"
+      target="${javac.version}"
+      source="${javac.version}"
+      deprecation="${javac.deprecation}">
+        <classpath refid="classpath" />
+    </iajc>
+    <echo message="Weaving of aspects is finished"/>
+  </target>
+
   <target name="compile-core" depends="clover, compile-hdfs-classes" description="Compile"/> 
 
   <target name="compile-contrib" depends="compile-core">
@@ -449,6 +474,8 @@
     <mkdir dir="${test.log.dir}"/>
     <copy file="${test.src.dir}/hadoop-policy.xml" 
       todir="${test.build.extraconf}" />
+    <copy file="${test.src.dir}/fi-site.xml"
+      todir="${test.build.extraconf}" />
     <junit showoutput="${test.output}"
       printsummary="${test.junit.printsummary}"
       haltonfailure="${test.junit.haltonfailure}"
@@ -465,6 +492,10 @@
       <sysproperty key="test.build.extraconf" value="${test.build.extraconf}" />
       <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
       <classpath refid="test.classpath"/>
+      <!-- Pass probability specifications to the spawn JVM -->
+      <syspropertyset id="FaultProbabilityProperties">
+        <propertyref regex="fi.*"/>
+      </syspropertyset>
       <formatter type="${test.junit.output.format}" />
       <batchtest todir="${test.build.dir}" unless="testcase">
         <fileset dir="${test.src.dir}/hdfs"
@@ -486,6 +517,8 @@
     <mkdir dir="${test.log.dir}"/>
     <copy file="${test.src.dir}/hadoop-policy.xml" 
       todir="${test.build.extraconf}" />
+    <copy file="${test.src.dir}/fi-site.xml"
+      todir="${test.build.extraconf}" />
     <junit showoutput="${test.output}"
       printsummary="${test.junit.printsummary}"
       haltonfailure="${test.junit.haltonfailure}"

ivy.xml

@@ -265,6 +265,16 @@
       rev="${slf4j-log4j12.version}"
       conf="common->master">
     </dependency>
+    <dependency org="org.aspectj"
+      name="aspectjrt"
+      rev="${aspectj.version}"
+      conf="common->default">
+    </dependency>
+    <dependency org="org.aspectj"
+      name="aspectjtools"
+      rev="${aspectj.version}"
+      conf="common->default">
+    </dependency>
     </dependencies>
   
 </ivy-module>

ivy/libraries.properties

@@ -70,3 +70,5 @@ slf4j-log4j12.version=1.4.3
 
 xmlenc.version=0.52
 xerces.version=1.4.4
+
+aspectj.version=1.6.4

src/test/aop/org/apache/hadoop/fi/FiConfig.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fi;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class wraps the logic around fault injection configuration file
+ * Default file is expected to be found in src/test/fi-site.xml
+ * This default file should be copied by JUnit Ant's tasks to 
+ * build/test/extraconf folder before tests are ran
+ * An alternative location can be set through
+ *   -Dfi.config=<file_name>
+ */
+public class FiConfig {
+  private static final String CONFIG_PARAMETER = ProbabilityModel.FPROB_NAME + "config";
+  private static final String DEFAULT_CONFIG = "fi-site.xml";
+  private static Configuration conf;
+  static {
+    init();
+  }
+  
+  protected static void init () {
+    if (conf == null) {
+      conf = new Configuration(false);
+      System.out.println(System.getProperties());
+      String configName = System.getProperty(CONFIG_PARAMETER, DEFAULT_CONFIG);
+      conf.addResource(configName);
+    }
+  }
+  
+  /**
+   * Method provides access to local Configuration 
+   * 
+   * @return Configuration initialized with fault injection's parameters
+   */
+  public static Configuration getConfig() {
+    return conf;
+  }
+}

src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fi;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class is responsible for the decision of when a fault 
+ * has to be triggered within a class of Hadoop
+ * 
+ *  Default probability of injection is set to 0%. To change it
+ *  one can set the sys. prop. -Dfi.*=<new probability level>
+ *  Another way to do so is to set this level through FI config file,
+ *  located under src/test/fi-site.conf
+ *  
+ *  To change the level one has to specify the following sys,prop.:
+ *  -Dfi.<name of fault location>=<probability level> in the runtime
+ *  Probability level is specified by a float between 0.0 and 1.0
+ *  
+ *  <name of fault location> might be represented by a short classname
+ *  or otherwise. This decision is left up to the discretion of aspects
+ *  developer, but has to be consistent through the code 
+ */
+public class ProbabilityModel {
+  private static Random generator = new Random();
+  private static final Log LOG = LogFactory.getLog(ProbabilityModel.class);
+
+  static final String FPROB_NAME = "fi.";
+  private static final String ALL_PROBABILITIES = FPROB_NAME + "*";
+  private static final float DEFAULT_PROB = 0.00f; // Default probability rate is 0%
+
+  private static Configuration conf = FiConfig.getConfig();
+
+  static {
+    // Set new default probability if specified through a system.property
+    // If neither is specified set default probability to DEFAULT_PROB 
+    conf.set(ALL_PROBABILITIES, 
+        System.getProperty(ALL_PROBABILITIES, 
+            conf.get(ALL_PROBABILITIES, Float.toString(DEFAULT_PROB))));
+
+    LOG.info(ALL_PROBABILITIES + "=" + conf.get(ALL_PROBABILITIES));
+  }
+
+  // Simplistic method to check if we have reached the point of injection
+  public static boolean injectCriteria(String klassName) {
+    boolean trigger = false;
+    // TODO fix this: make it more sophisticated!!!
+    if (generator.nextFloat() < getProbability(klassName)) {
+      trigger = true;
+    }
+    return trigger;
+  }
+
+  // This primitive checks for arbitrary set of desired probability and
+  // uses default setting if it wasn't
+  // The probability expected to be set as an float between 0 and 100
+  protected static float getProbability(final String klass) {
+    String newProbName = FPROB_NAME + klass;
+
+    conf.setIfUnset(newProbName, System.getProperty(newProbName, conf.get(ALL_PROBABILITIES)));
+    float ret = conf.getFloat(newProbName, conf.getFloat(ALL_PROBABILITIES, DEFAULT_PROB));
+    LOG.debug("Request for " + newProbName + " returns=" + ret);
+    // Make sure that probability level is valid.
+    if (ret < 0.00 || ret > 1.00) 
+      ret = conf.getFloat(ALL_PROBABILITIES, DEFAULT_PROB);
+    
+    return ret;
+  }
+}

src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fi.ProbabilityModel;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.util.DiskChecker.*;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.DataOutputStream;
+
+/**
+ * This aspect takes care about faults injected into datanode.BlockReceiver 
+ * class 
+ */
+public aspect BlockReceiverAspects {
+  public static final Log LOG = LogFactory.getLog(BlockReceiverAspects.class);
+
+  pointcut callReceivePacket() :
+    call (* OutputStream.write(..))
+      && withincode (* BlockReceiver.receivePacket(..))
+// to further limit the application of this aspect a very narrow 'target' can be used as follows
+//  && target(DataOutputStream)
+      && !within(BlockReceiverAspects +);
+	
+  before () throws IOException : callReceivePacket () {
+    if (ProbabilityModel.injectCriteria(BlockReceiver.class.getSimpleName())) {
+      LOG.info("Before the injection point");
+      Thread.dumpStack();
+      throw new DiskOutOfSpaceException ("FI: injected fault point at " + 
+        thisJoinPoint.getStaticPart( ).getSourceLocation());
+    }
+  }
+}

src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fi.ProbabilityModel;
+import org.apache.hadoop.hdfs.server.datanode.FSDataset;
+import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface.BlockWriteStreams;
+import org.apache.hadoop.util.DiskChecker.*;
+
+
+/**
+ * This aspect takes care about faults injected into datanode.FSDatase class
+ */
+public aspect FSDatasetAspects {
+  public static final Log LOG = LogFactory.getLog(BlockReceiverAspects.class);
+
+  pointcut execGetBlockFile() :
+  // the following will inject faults inside of the method in question 		
+    execution (* FSDataset.getBlockFile(..)) && !within(FSDatasetAspects +);
+
+  // the following will inject faults before the actual call of the method
+  // call (* FSDataset.getBlockFile(..)) && !within(FSDatasetAspects +);
+
+  pointcut callCreateBlockWriteStream() : 
+    call (BlockWriteStreams FSDataset.createBlockWriteStreams(..)) 
+      && !within(FSDatasetAspects +);
+
+  // This aspect specifies the logic of our fault point.
+  // In this case it simply throws DiskErrorException at the very beginning of
+  // invocation of the method, specified by callGetBlockFile() pointcut
+  before() throws DiskErrorException : execGetBlockFile() {
+    if (ProbabilityModel.injectCriteria(FSDataset.class.getSimpleName())) {
+      LOG.info("Before the injection point");
+      Thread.dumpStack();
+      throw new DiskErrorException("FI: injected fault point at "
+          + thisJoinPoint.getStaticPart().getSourceLocation());
+    }
+  }
+
+  before() throws DiskOutOfSpaceException : callCreateBlockWriteStream() {
+    if (ProbabilityModel.injectCriteria(FSDataset.class.getSimpleName())) {
+      LOG.info("Before the injection point");
+      Thread.dumpStack();
+      throw new DiskOutOfSpaceException("FI: injected fault point at "
+          + thisJoinPoint.getStaticPart().getSourceLocation());
+    }
+  }
+}

src/test/fi-site.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put fault injection specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>fi.*</name>
+    <value>0.00</value>
+    <description>
+    	Default probability level for all injected faults specified 
+    	as a floating number between 0 and 1.00
+    </description>
+  </property>
+</configuration>