hadoop/build.xml

<?xml version="1.0"?>

<project name="Hadoop" default="compile">

  <!-- Load all the default properties, and any the user wants    -->
  <!-- to contribute (without having to type -D or edit this file -->
  <property file="${user.home}/build.properties" />
  <property file="${basedir}/build.properties" />
 
  <property name="Name" value="Hadoop"/>
  <property name="name" value="hadoop"/>
  <property name="version" value="0.9.3-dev"/>
  <property name="final.name" value="${name}-${version}"/>
  <property name="year" value="2006"/>
  <property name="libhdfs.version" value="1"/>

  <property name="src.dir" value="${basedir}/src/java"/>
  <property name="native.src.dir" value="${basedir}/src/native"/>
  <property name="examples.dir" value="${basedir}/src/examples"/>
  <property name="lib.dir" value="${basedir}/lib"/>
  <property name="conf.dir" value="${basedir}/conf"/>
  <property name="docs.dir" value="${basedir}/docs"/>
  <property name="docs.src" value="${basedir}/src/web"/>
  <property name="libhdfs.src" value="${basedir}/src/c++/libhdfs"/>

  <property name="build.dir" value="${basedir}/build"/>
  <property name="build.classes" value="${build.dir}/classes"/>
  <property name="build.src" value="${build.dir}/src"/>
  <property name="build.webapps" value="${build.dir}/webapps"/>
  <property name="build.examples" value="${build.dir}/examples"/>
  <property name="build.libhdfs" value="${build.dir}/libhdfs"/>
  <property name="build.platform" value="${os.name}-${os.arch}-${sun.arch.data.model}"/>
  <property name="build.native" value="${build.dir}/native/${build.platform}"/>
  <property name="build.docs" value="${build.dir}/docs"/>
  <property name="build.javadoc" value="${build.docs}/api"/>
  <property name="build.encoding" value="ISO-8859-1"/>

  <property name="test.src.dir" value="${basedir}/src/test"/>
  <property name="test.build.dir" value="${build.dir}/test"/>
  <property name="test.build.data" value="${test.build.dir}/data"/>
  <property name="test.cache.data" value="${test.build.dir}/cache"/>
  <property name="test.log.dir" value="${test.build.dir}/logs"/>
  <property name="test.build.classes" value="${test.build.dir}/classes"/>
  <property name="test.build.testjar" value="${test.build.dir}/testjar"/>
  <property name="test.build.javadoc" value="${test.build.dir}/docs/api"/>
  <property name="test.include" value="Test*"/>
  <property name="test.classpath.id" value="test.classpath"/>
  <property name="test.output" value="no"/>
  <property name="test.junit.output.format" value="plain"/>

  <property name="libhdfs.test.conf.dir" value="${libhdfs.src}/tests/conf"/>
  <property name="libhdfs.test.dir" value="${test.build.dir}/libhdfs"/>

  <property name="web.src.dir" value="${basedir}/src/web"/>
  <property name="src.webapps" value="${basedir}/src/webapps"/>

  <property name="javadoc.link.java"
	    value="http://java.sun.com/j2se/1.5/docs/api/"/>
  <property name="javadoc.packages" value="org.apache.hadoop.*"/>

  <property name="dist.dir" value="${build.dir}/${final.name}"/>

  <property name="javac.debug" value="on"/>
  <property name="javac.optimize" value="on"/>
  <property name="javac.deprecation" value="off"/>
  <property name="javac.version" value="1.5"/>

  <!-- the normal classpath -->
  <path id="classpath">
    <pathelement location="${build.classes}"/>
    <fileset dir="${lib.dir}">
      <include name="**/*.jar" />
    </fileset>
    <pathelement location="${conf.dir}"/>
  </path>

  <!-- the unit test classpath: uses test.src.dir for configuration -->
  <path id="test.classpath">
    <pathelement location="${test.build.classes}" />
    <pathelement location="${test.src.dir}"/>
    <pathelement location="${build.dir}"/>
    <pathelement location="${build.examples}"/>
    <path refid="classpath"/>
  </path>

  <!-- the cluster test classpath: uses conf.dir for configuration -->
  <path id="test.cluster.classpath">
    <path refid="classpath"/>
    <pathelement location="${test.build.classes}" />
    <pathelement location="${test.src.dir}"/>
    <pathelement location="${build.dir}"/>
  </path>

  <!-- ====================================================== -->
  <!-- Stuff needed by all targets                            -->
  <!-- ====================================================== -->
  <target name="init">
    <mkdir dir="${build.dir}"/>
    <mkdir dir="${build.classes}"/>
    <mkdir dir="${build.src}"/>
    <mkdir dir="${build.webapps}/task/WEB-INF"/>
    <mkdir dir="${build.webapps}/job/WEB-INF"/>
    <mkdir dir="${build.webapps}/dfs/WEB-INF"/>
    <mkdir dir="${build.webapps}/datanode/WEB-INF"/>
    <mkdir dir="${build.examples}"/>
 
    <mkdir dir="${test.build.dir}"/>
    <mkdir dir="${test.build.classes}"/>
    <mkdir dir="${test.build.testjar}"/>
    <touch datetime="01/25/1971 2:00 pm">
      <fileset dir="${conf.dir}" includes="**/*.template"/>
    </touch>

    <!-- copy all of the jsp and static files -->
    <copy todir="${build.webapps}">
      <fileset dir="${src.webapps}">
        <exclude name="**/*.jsp" />
      </fileset>
    </copy>

    <copy todir="${conf.dir}" verbose="true">
      <fileset dir="${conf.dir}" includes="**/*.template"/>
      <mapper type="glob" from="*.template" to="*"/>
    </copy>

    <exec executable="sh">
       <arg line="src/saveVersion.sh ${version}"/>
    </exec>
  </target>

  <!-- ====================================================== -->
  <!-- Compile the Java files                                 -->
  <!-- ====================================================== -->
  <taskdef classname="org.apache.jasper.JspC" name="jsp-compile" >
      <classpath refid="test.classpath"/>
  </taskdef>

  <target name="record-parser" depends="init" if="javacc.home">
      <javacc
          target="${src.dir}/org/apache/hadoop/record/compiler/generated/rcc.jj"
          outputdirectory="${src.dir}/org/apache/hadoop/record/compiler/generated"
          javacchome="${javacc.home}"
      />
  </target>
  
  <target name="compile-core-classes" depends="init, record-parser">

    <copy file="${src.webapps}/datanode/browseDirectory.jsp" todir="${src.webapps}/dfs/"/>
    <jsp-compile
     uriroot="${src.webapps}/task"
     outputdir="${build.src}"
     package="org.apache.hadoop.mapred"
     webxml="${build.webapps}/task/WEB-INF/web.xml">
    </jsp-compile>

    <jsp-compile
     uriroot="${src.webapps}/job"
     outputdir="${build.src}"
     package="org.apache.hadoop.mapred"
     webxml="${build.webapps}/job/WEB-INF/web.xml">
    </jsp-compile>

    <jsp-compile
     uriroot="${src.webapps}/dfs"
     outputdir="${build.src}"
     package="org.apache.hadoop.dfs"
     webxml="${build.webapps}/dfs/WEB-INF/web.xml">
    </jsp-compile>

    <jsp-compile
     uriroot="${src.webapps}/datanode"
     outputdir="${build.src}"
     package="org.apache.hadoop.dfs"
     webxml="${build.webapps}/datanode/WEB-INF/web.xml">
    </jsp-compile>

    <javac 
     encoding="${build.encoding}" 
     srcdir="${src.dir};${build.src}"
     includes="org/apache/hadoop/**/*.java"
     destdir="${build.classes}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <classpath refid="classpath"/>
    </javac>    

    </target>

  <target name="compile-core-native" depends="compile-core-classes"
          if="compile.native">
  	
    <mkdir dir="${build.native}/lib"/>
    <mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/zlib"/>

  	<javah
  	  classpath="${build.classes}"
  	  destdir="${build.native}/src/org/apache/hadoop/io/compress/zlib"
      force="yes"
  	  verbose="yes"
  	  >
  	  <class name="org.apache.hadoop.io.compress.zlib.ZlibCompressor" />
      <class name="org.apache.hadoop.io.compress.zlib.ZlibDecompressor" />
  	</javah>

	<exec dir="${build.native}" executable="sh" failonerror="true">
	  <env key="OS_NAME" value="${os.name}"/>
	  <env key="OS_ARCH" value="${os.arch}"/>
	  <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}"/>
	  <env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}"/>
	  <arg line="${native.src.dir}/configure"/>
    </exec>

    <exec dir="${build.native}" executable="make" failonerror="true">
      <env key="OS_NAME" value="${os.name}"/>
      <env key="OS_ARCH" value="${os.arch}"/>
  	  <env key="JVM_DATA_MODEL" value="${sun.arch.data.model}"/>
  	  <env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}"/>
    </exec>

	<exec dir="${build.native}" executable="sh" failonerror="true">
	  <arg line="${build.native}/libtool --mode=install cp ${build.native}/lib/libhadoop.la ${build.native}/lib"/>
    </exec>

  </target>

  <target name="compile-core" depends="compile-core-classes,compile-core-native">
  </target>

  <target name="compile-contrib" depends="compile-core">
     <subant target="deploy">
        <fileset file="${basedir}/src/contrib/build.xml"/>
     </subant>  	
  </target>
  
  <target name="compile" depends="compile-core, compile-contrib">
  </target>

  <target name="compile-examples" depends="compile-core">
    <javac 
     encoding="${build.encoding}" 
     srcdir="${examples.dir}"
     includes="org/apache/hadoop/**/*.java"
     destdir="${build.examples}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <classpath refid="classpath"/>
    </javac>    
  </target>

  <!-- ================================================================== -->
  <!-- Make hadoop.jar                                                     -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="jar" depends="compile-core">
    <tar compression="gzip" destfile="${build.classes}/bin.tgz">
      <tarfileset dir="bin" mode="755"/>
    </tar>
    <jar jarfile="${build.dir}/${final.name}.jar"
         basedir="${build.classes}">
      <manifest>
        <section name="org/apache/hadoop">
          <attribute name="Implementation-Title" value="Hadoop"/>
          <attribute name="Implementation-Version" value="${version}"/>
          <attribute name="Implementation-Vendor" value="Apache"/>
        </section>
      </manifest>
      <fileset file="${conf.dir}/hadoop-default.xml"/>
      <fileset file="${conf.dir}/mapred-default.xml"/>
      <fileset file="${conf.dir}/commons-logging.properties"/>
      <fileset file="${conf.dir}/log4j.properties"/>
      <fileset file="${conf.dir}/hadoop-metrics.properties"/>
      <zipfileset dir="${build.webapps}" prefix="webapps"/>
    </jar>
  </target>

  <!-- ================================================================== -->
  <!-- Make the Hadoop examples jar.                                      -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="examples" depends="jar, compile-examples">
    <jar jarfile="${build.dir}/${final.name}-examples.jar"
         basedir="${build.examples}">
      <manifest>
        <attribute name="Main-Class" 
                   value="org/apache/hadoop/examples/ExampleDriver"/>
      </manifest>
    </jar>
  </target>

  <!-- ================================================================== -->
  <!-- Make the Hadoop metrics jar. (for use outside Hadoop)              -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="metrics.jar" depends="compile-core">
    <jar jarfile="${build.dir}/hadoop-metrics-${version}.jar"
         basedir="${build.classes}">
      <include name="**/metrics/**" />
      <exclude name="**/package.html" />
    </jar>
  </target>


  <!-- ================================================================== -->
  <!-- Compile test code                                                  --> 
  <!-- ================================================================== -->
  <target name="compile-core-test" depends="compile-examples">
    <javac 
     encoding="${build.encoding}" 
     srcdir="${test.src.dir}"
     includes="org/apache/hadoop/**/*.java"
     destdir="${test.build.classes}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <classpath refid="test.classpath"/>
    </javac> 
    <javac
     encoding="${build.encoding}"
     srcdir="${test.src.dir}/testjar"
     includes="*.java"
     destdir="${test.build.testjar}"
     debug="${javac.debug}"
     optimize="${javac.optimize}"
     target="${javac.version}"
     source="${javac.version}"
     deprecation="${javac.deprecation}">
      <classpath refid="test.classpath"/>
    </javac>                                 
    <delete file="${test.build.testjar}/testjob.jar"/> 
    <jar jarfile="${test.build.testjar}/testjob.jar"
     basedir="${test.build.testjar}">
    </jar>
                                                              
    <jar jarfile="${build.dir}/${final.name}-test.jar"
         basedir="${test.build.classes}">
         <manifest>
           <attribute name="Main-Class"
                      value="org/apache/hadoop/test/AllTestDriver"/>
         </manifest>
    </jar>
    <delete dir="${test.cache.data}"/>
    <mkdir dir="${test.cache.data}"/>
    <copy file="${test.src.dir}/org/apache/hadoop/mapred/test.txt" todir="${test.cache.data}"/>
    <copy file="${test.src.dir}/org/apache/hadoop/mapred/test.jar" todir="${test.cache.data}"/>
    <copy file="${test.src.dir}/org/apache/hadoop/mapred/test.zip" todir="${test.cache.data}"/>
  </target>

  <!-- ================================================================== -->
  <!-- Run unit tests                                                     --> 
  <!-- ================================================================== -->
  <target name="test-core" depends="compile-core-test">

    <delete dir="${test.build.data}"/>
    <mkdir dir="${test.build.data}"/>
    <delete dir="${test.log.dir}"/>
    <mkdir dir="${test.log.dir}"/>
    <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" 
           fork="yes" dir="${basedir}"
      errorProperty="tests.failed" failureProperty="tests.failed">
      <sysproperty key="test.build.data" value="${test.build.data}"/>
      <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
      <sysproperty key="test.src.dir" value="${test.src.dir}"/>
      <sysproperty key="java.library.path"
       value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
      <classpath refid="${test.classpath.id}"/>
      <formatter type="${test.junit.output.format}" />
      <batchtest todir="${test.build.dir}" unless="testcase">
        <fileset dir="${test.src.dir}"
	         includes="**/${test.include}.java"
		 excludes="**/${test.exclude}.java" />
      </batchtest>
      <batchtest todir="${test.build.dir}" if="testcase">
        <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
      </batchtest>
    </junit>


    <fail if="tests.failed">Tests failed!</fail>

  </target>   

  <target name="test-contrib" depends="compile-core, compile-core-test">
    <subant target="test">
       <fileset file="${basedir}/src/contrib/build.xml"/>
    </subant> 
  </target>   
  
  <target name="test" depends="test-core, test-contrib">
  </target>

  <!-- Run all unit tests, not just Test*, and use non-test configuration. -->
  <target name="test-cluster">
    <antcall target="test">
      <param name="test.include" value="*"/>
      <param name="test.classpath.id" value="test.cluster.classpath"/>
    </antcall>
  </target>

  <target name="nightly" depends="test, tar">
  </target>

  <!-- ================================================================== -->
  <!-- Documentation                                                      -->
  <!-- ================================================================== -->
  <target name="javadoc" depends="default-doc">
    <mkdir dir="${build.javadoc}"/>
    <javadoc
      overview="${src.dir}/overview.html"
      destdir="${build.javadoc}"
      author="true"
      version="true"
      use="true"
      windowtitle="${Name} ${version} API"
      doctitle="${Name} ${version} API"
      bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
      >
    	<packageset dir="${src.dir}"/>
    	<packageset dir="${examples.dir}"/>
        <link href="${javadoc.link.java}"/>
        <classpath refid="classpath"/>

    	<group title="Core" packages="org.apache.hadoop.*"/>
    	<group title="Examples" packages="org.apache.hadoop.examples"/>

    </javadoc>
  </target>	
	
  <target name="default-doc">
    <style basedir="${conf.dir}" destdir="${build.docs}"
           includes="hadoop-default.xml" style="conf/configuration.xsl"/>
  </target>

  <!-- ================================================================== -->
  <!-- D I S T R I B U T I O N                                            -->
  <!-- ================================================================== -->
  <!--                                                                    -->
  <!-- ================================================================== -->
  <target name="package" depends="jar, javadoc, examples, compile-core-test">
    <mkdir dir="${dist.dir}"/>
    <mkdir dir="${dist.dir}/lib"/>
    <mkdir dir="${dist.dir}/bin"/>
    <mkdir dir="${dist.dir}/docs"/>
    <mkdir dir="${dist.dir}/docs/api"/>

    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
      <fileset dir="lib">
        <exclude name="**/native/**"/>
      </fileset>
    </copy>

  	<exec dir="${dist.dir}" executable="sh" failonerror="true">
	  <env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
	  <env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
	  <env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
	  <arg line="${native.src.dir}/packageNativeHadoop.sh"/>
    </exec>

    <copy todir="${dist.dir}/webapps">
      <fileset dir="${build.webapps}"/>
    </copy>

    <copy file="${build.dir}/${final.name}.jar" todir="${dist.dir}"/>

    <copy file="${build.dir}/${final.name}-examples.jar" todir="${dist.dir}"/>
    
    <copy file="${build.dir}/${final.name}-test.jar" todir="${dist.dir}"/>

    <copy todir="${dist.dir}/bin">
      <fileset dir="bin"/>
    </copy>

    <copy todir="${dist.dir}/conf">
      <fileset dir="${conf.dir}" excludes="**/*.template"/>
    </copy>

    <chmod perm="ugo+x" type="file">
        <fileset dir="${dist.dir}/bin"/>
    </chmod>

    <copy todir="${dist.dir}/docs">
      <fileset dir="${build.docs}"/>
    </copy>

    <copy todir="${dist.dir}">
      <fileset dir=".">
        <include name="*.txt" />
      </fileset>
    </copy>

    <copy todir="${dist.dir}/src" includeEmptyDirs="true">
      <fileset dir="src"/>
    </copy>

    <copy todir="${dist.dir}/" file="build.xml"/>

  </target>

  <!-- ================================================================== -->
  <!-- Make release tarball                                               -->
  <!-- ================================================================== -->
  <target name="tar" depends="package">
    <tar compression="gzip" longfile="gnu"
      destfile="${build.dir}/${final.name}.tar.gz">
      <tarfileset dir="${build.dir}" mode="664">
	<exclude name="${final.name}/bin/*" />
        <include name="${final.name}/**" />
      </tarfileset>
      <tarfileset dir="${build.dir}" mode="755">
        <include name="${final.name}/bin/*" />
      </tarfileset>
    </tar>
  </target>
	
  <!-- ================================================================== -->
  <!-- Clean.  Delete the build files, and their directories              -->
  <!-- ================================================================== -->
  <target name="clean" depends="clean-contrib">
  	<delete dir="${build.dir}"/>
  </target>

  <!-- ================================================================== -->
  <!-- Contrib targets. For now, they must be called explicitely          -->
  <!-- Using subant instead of ant as a workaround for 30569              -->
  <!-- ================================================================== -->
  <target name="deploy-contrib" depends="compile-core">
     <subant target="deploy">        
        <fileset file="src/contrib/build.xml"/>
     </subant>  	
  </target>
  <target name="clean-contrib">
     <subant target="clean">        
        <fileset file="src/contrib/build.xml"/>
     </subant>  	
  </target>

  <!-- ================================================================== -->
  <!-- libhdfs targets.                                                   -->
  <!-- ================================================================== -->
  <target name="compile-libhdfs" depends="init">
    <mkdir dir="${build.libhdfs}"/>
    <exec dir="${libhdfs.src}" executable="make">
      <env key="OS_NAME" value="${os.name}"/>
      <env key="OS_ARCH" value="${os.arch}"/>
      <env key="SHLIB_VERSION" value="${libhdfs.version}"/>
      <env key="LIBHDFS_BUILD_DIR" value="${build.libhdfs}"/>
    </exec>
  </target>
	
  <target name="test-libhdfs" depends="compile-libhdfs, compile-core">
    <delete dir="${libhdfs.test.dir}"/>
    <mkdir dir="${libhdfs.test.dir}"/>
    <mkdir dir="${libhdfs.test.dir}/logs"/>
    <mkdir dir="${libhdfs.test.dir}/dfs/name"/>

    <exec dir="${libhdfs.src}" executable="make" failonerror="true">
        <env key="OS_NAME" value="${os.name}"/>
        <env key="OS_ARCH" value="${os.arch}"/>
        <env key="SHLIB_VERSION" value="${libhdfs.version}"/>
        <env key="LIBHDFS_BUILD_DIR" value="${build.libhdfs}"/>
        <env key="HADOOP_HOME" value="${basedir}"/>
        <env key="HADOOP_CONF_DIR" value="${libhdfs.test.conf.dir}"/>
        <env key="HADOOP_LOG_DIR" value="${libhdfs.test.dir}/logs"/>
		<arg value="test"/>
    </exec>
  </target>
	
  <target name="clean-libhdfs">
    <exec dir="${libhdfs.src}" executable="make">
      <env key="LIBHDFS_BUILD_DIR" value="${build.libhdfs}"/>
    	<arg value="clean"/>
    </exec>
  </target>

  <target name="doc-libhdfs" depends="compile-libhdfs">
	<exec dir="${libhdfs.src}" executable="make">
	  <arg value="doc"/>
	</exec>
  </target>
	
  <target name="package-libhdfs" depends="compile-libhdfs, doc-libhdfs, package">
  	<mkdir dir="${dist.dir}/libhdfs"/>
  	<copy todir="${dist.dir}/libhdfs">
  	  <fileset dir="${build.libhdfs}" 
  	  	casesensitive="yes" 
  	  	followsymlinks="false">
  	  	<exclude name="**/tests/**"/>
  	  	<exclude name="*.so"/>
  	  	<exclude name="*.o"/>
  	  </fileset>
  	</copy>
  	<chmod perm="ugo+x" type="file">
  	    <fileset dir="${dist.dir}/libhdfs"/>
  	</chmod>
  	<exec dir="${dist.dir}/libhdfs" executable="ln">
  		<arg line="-sf libhdfs.so.${libhdfs.version} libhdfs.so"/>
  	</exec>
  </target>
		
</project>