Merging trunk changes to branch-trunk-win.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-trunk-win@1401071 13f79535-47bb-0310-9956-ffa450edef68

.gitattributes

@@ -0,0 +1,18 @@
+# Auto detect text files and perform LF normalization
+*        text=auto
+
+*.cs     text diff=csharp
+*.java   text diff=java
+*.html   text diff=html
+*.py     text diff=python
+*.pl     text diff=perl
+*.pm     text diff=perl
+*.css    text
+*.js     text
+*.sql    text
+
+*.sh     text eol=lf
+
+*.bat    text eol=crlf
+*.csproj text merge=union eol=crlf
+*.sln    text merge=union eol=crlf

dev-support/test-patch.sh

@@ -32,7 +32,7 @@ JENKINS=false
 PATCH_DIR=/tmp
 SUPPORT_DIR=/tmp
 BASEDIR=$(pwd)
-
+BUILD_NATIVE=true
 PS=${PS:-ps}
 AWK=${AWK:-awk}
 WGET=${WGET:-wget}
@@ -67,6 +67,7 @@ printUsage() {
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
   echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
   echo "--run-tests            Run all tests below the base directory"
+  echo "--build-native=<bool>  If true, then build native components (default 'true')"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -139,11 +140,18 @@ parseArgs() {
     --run-tests)
       RUN_TESTS=true
       ;;
+    --build-native=*)
+      BUILD_NATIVE=${i#*=}
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
     esac
   done
+  if [[ $BUILD_NATIVE == "true" ]] ; then
+    NATIVE_PROFILE=-Pnative
+    REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
+  fi
   if [ -z "$PATCH_OR_DEFECT" ]; then
     printUsage
     exit 1
@@ -437,8 +445,8 @@ checkJavacWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
-  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
+  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
+  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
@@ -707,8 +715,8 @@ runTests () {
       if [[ $building_common -eq 0 ]]; then
           echo "  Building hadoop-common with -Pnative in order to provide \
 libhadoop.so to the hadoop-hdfs unit tests."
-          echo "  $MVN compile -Pnative -D${PROJECT_NAME}PatchProcess"
-          if ! $MVN compile -Pnative -D${PROJECT_NAME}PatchProcess; then
+          echo "  $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess"
+          if ! $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess; then
               JIRA_COMMENT="$JIRA_COMMENT
         {color:red}-1 core tests{color}.  Failed to build the native portion \
 of hadoop-common prior to running the unit tests in $ordered_modules"
@@ -719,8 +727,8 @@ of hadoop-common prior to running the unit tests in $ordered_modules"
   for module in $ordered_modules; do
     cd $module
     echo "  Running tests in $module"
-    echo "  $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess"
-    $MVN clean install -fn -Pnative -Drequire.test.libhadoop -D${PROJECT_NAME}PatchProcess
+    echo "  $MVN clean install -fn $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess"
+    $MVN clean install -fn $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess
     module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
     # With -fn mvn always exits with a 0 exit code.  Because of this we need to
     # find the errors instead of using the exit code.  We assume that if the build
@@ -763,7 +771,7 @@ findModule (){
 findModules () {
   # Come up with a list of changed files into $TMP
   TMP=/tmp/tmp.paths.$$
-  $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP
+  $GREP '^+++ \|^--- ' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP
   
   # if all of the lines start with a/ or b/, then this is a git patch that
   # was generated without --no-prefix

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -158,7 +158,7 @@ public class KerberosAuthenticator implements Authenticator {
       conn.setRequestMethod(AUTH_HTTP_METHOD);
       conn.connect();
       
-      if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
+      if (conn.getRequestProperty(AUTHORIZATION) != null && conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
         LOG.debug("JDK performed authentication on our behalf.");
         // If the JDK already did the SPNEGO back-and-forth for
         // us, just pull out the token.

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java

@@ -134,9 +134,11 @@ public abstract class AuthenticatorTestCase extends TestCase {
     try {
       URL url = new URL(getBaseURL());
       AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      Assert.assertFalse(token.isSet());
       TestConnectionConfigurator connConf = new TestConnectionConfigurator();
       AuthenticatedURL aUrl = new AuthenticatedURL(authenticator, connConf);
       HttpURLConnection conn = aUrl.openConnection(url, token);
+      Assert.assertTrue(token.isSet());
       Assert.assertTrue(connConf.invoked);
       String tokenStr = token.toString();
       if (doPost) {

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java

@@ -43,6 +43,14 @@ public class TestKerberosAuthenticator extends AuthenticatorTestCase {
     _testAuthentication(new KerberosAuthenticator(), false);
   }
 
+  public void testFallbacktoPseudoAuthenticatorAnonymous() throws Exception {
+    Properties props = new Properties();
+    props.setProperty(AuthenticationFilter.AUTH_TYPE, "simple");
+    props.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "true");
+    setAuthenticationHandlerConfig(props);
+    _testAuthentication(new KerberosAuthenticator(), false);
+  }
+
   public void testNotAuthenticated() throws Exception {
     setAuthenticationHandlerConfig(getAuthenticationHandlerConfiguration());
     start();

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -72,8 +72,8 @@ Trunk (Unreleased)
     HADOOP-8360. empty-configuration.xml fails xml validation
     (Radim Kolar via harsh)
 
-    HADOOP-8367 Improve documentation of declaringClassProtocolName in rpc headers 
-                (Sanjay Radia)
+    HADOOP-8367 Improve documentation of declaringClassProtocolName in 
+    rpc headers. (Sanjay Radia)
 
     HADOOP-8415. Add getDouble() and setDouble() in
     org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
@@ -122,6 +122,13 @@ Trunk (Unreleased)
     HADOOP-8864. Addendum to HADOOP-8840: Add a coloring case for +0 results
     too. (harsh)
 
+    HADOOP-8910. Add examples to GlobExpander#expand method. (suresh)
+
+    HADOOP-8920. Add more javadoc to metrics2 related classes. (suresh)
+
+    HADOOP-8776. Provide an option in test-patch that can enable/disable
+    compiling native code. (Chris Nauroth via suresh)
+
   BUG FIXES
 
     HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -259,6 +266,9 @@ Trunk (Unreleased)
     HADOOP-8839. test-patch's -1 on @author tag presence doesn't cause
     a -1 to the overall result (harsh)
 
+    HADOOP-8918. test-patch.sh is parsing modified files wrong.
+    (Raja Aluri via suresh)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -303,11 +313,31 @@ Release 2.0.3-alpha - Unreleased
     HADOOP-8909. Hadoop Common Maven protoc calls must not depend on external
     sh script. (Chris Nauroth via suresh)
 
+    HADOOP-8911. CRLF characters in source and text files.
+    (Raja Aluri via suresh)
+
+    HADOOP-8912. Add .gitattributes file to prevent CRLF and LF mismatches
+    for source and text files. (Raja Aluri via suresh)
+
+    HADOOP-8784. Improve IPC.Client's token use (daryn)
+
+    HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+
+    HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
+    javascript in browser dashboard (Damien Hardy via bobby)
+
+    HADOOP-8931. Add Java version to startup message. (eli)
+
+    HADOOP-8925. Remove the packaging. (eli)
+
   OPTIMIZATIONS
 
     HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
     via atm)
 
+    HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+    data (Gopal V via bobby)
+
   BUG FIXES
 
     HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to
@@ -338,6 +368,20 @@ Release 2.0.3-alpha - Unreleased
     webhdfs filesystem and fsck to fail when security is on.
     (Arpit Gupta via suresh)
 
+    HADOOP-8901. GZip and Snappy support may not work without unversioned
+    libraries (Colin Patrick McCabe via todd)
+
+    HADOOP-8883. Anonymous fallback in KerberosAuthenticator is broken.
+    (rkanter via tucu)
+
+    HADOOP-8900. BuiltInGzipDecompressor throws IOException - stored gzip size
+    doesn't match decompressed size. (Slavik Krassovsky via suresh)
+
+    HADOOP-8948. TestFileUtil.testGetDU fails on Windows due to incorrect
+    assumption of line separator. (Chris Nauroth via suresh)
+
+    HADOOP-8951. RunJar to fail with user-comprehensible error 
+    message if jar missing. (stevel via suresh)
 
 Release 2.0.2-alpha - 2012-09-07 
 
@@ -1037,10 +1081,19 @@ Release 0.23.5 - UNRELEASED
 
   IMPROVEMENTS
 
+    HADOOP-8932. JNI-based user-group mapping modules can be too chatty on 
+    lookup failures. (Kihwal Lee via suresh)
+
+    HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via
+    bobby)
+
   OPTIMIZATIONS
 
   BUG FIXES
 
+    HADOOP-8906. paths with multiple globs are unreliable. (Daryn Sharp via
+    jlowe)
+
 Release 0.23.4 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -50,12 +50,41 @@ function(dual_output_directory TGT DIR)
     output_directory(${TGT}_static "${DIR}")
 endfunction(dual_output_directory TGT DIR)
 
+#
+# This macro alters the behavior of find_package and find_library.
+# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable. 
+# You should save that variable before calling this function and restore it
+# after you have accomplished your goal.
+#
+# The behavior is altered in two ways:
+# 1. We always find shared libraries, never static;
+# 2. We find shared libraries with the given version number.
+#
+# On Windows this function is a no-op.  Windows does not encode
+# version number information information into library path names.
+#
+macro(set_find_shared_library_version LVERS)
+    IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+        # Mac OS uses .dylib
+        SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
+    ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        # Windows doesn't support finding shared libraries by version.
+    ELSE()
+        # Most UNIX variants use .so
+        SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
+    ENDIF()
+endmacro(set_find_shared_library_version LVERS)
+
 if (NOT GENERATED_JAVAH)
     # Must identify where the generated headers have been placed
     MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
 endif (NOT GENERATED_JAVAH)
 find_package(JNI REQUIRED)
+
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1")
 find_package(ZLIB REQUIRED)
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
@@ -69,10 +98,13 @@ INCLUDE(CheckCSourceCompiles)
 CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
 CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
 
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1")
 find_library(SNAPPY_LIBRARY 
     NAMES snappy
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
           ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
 find_path(SNAPPY_INCLUDE_DIR 
     NAMES snappy.h
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include

hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html

@@ -15,8 +15,8 @@ These release notes include new developer and user-facing incompatibilities, fea
 <li> <a href="https://issues.apache.org/jira/browse/YARN-137">YARN-137</a>.
      Major improvement reported by Siddharth Seth and fixed by Siddharth Seth (scheduler)<br>
      <b>Change the default scheduler to the CapacityScheduler</b><br>
-     <blockquote>There's some bugs in the FifoScheduler atm - doesn't distribute tasks across nodes and some headroom (available resource) issues.
-That's not the best experience for users trying out the 2.0 branch. The CS with the default configuration of a single queue behaves the same as the FifoScheduler and doesn't have these issues.
+     <blockquote>There's some bugs in the FifoScheduler atm - doesn't distribute tasks across nodes and some headroom (available resource) issues.
+That's not the best experience for users trying out the 2.0 branch. The CS with the default configuration of a single queue behaves the same as the FifoScheduler and doesn't have these issues.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-108">YARN-108</a>.
      Critical bug reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
@@ -45,73 +45,73 @@ That's not the best experience for users trying out the 2.0 branch. The CS with
 <li> <a href="https://issues.apache.org/jira/browse/YARN-79">YARN-79</a>.
      Major bug reported by Bikas Saha and fixed by Vinod Kumar Vavilapalli (client)<br>
      <b>Calling YarnClientImpl.close throws Exception</b><br>
-     <blockquote>The following exception is thrown
-===========
-*org.apache.hadoop.HadoopIllegalArgumentException: Cannot close proxy - is not Closeable or does not provide closeable invocation handler class org.apache.hadoop.yarn.api.impl.pb.client.ClientRMProtocolPBClientImpl*
-	*at org.apache.hadoop.ipc.RPC.stopProxy(RPC.java:624)*
-	*at org.hadoop.yarn.client.YarnClientImpl.stop(YarnClientImpl.java:102)*
-	at org.apache.hadoop.yarn.applications.unmanagedamlauncher.UnmanagedAMLauncher.run(UnmanagedAMLauncher.java:336)
-	at org.apache.hadoop.yarn.applications.unmanagedamlauncher.TestUnmanagedAMLauncher.testDSShell(TestUnmanagedAMLauncher.java:156)
-	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
-	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
-	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
-	at java.lang.reflect.Method.invoke(Method.java:597)
-	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
-	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
-	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
-	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
-	at org.junit.runners.BlockJUnit4ClassRunner.runNotIgnored(BlockJUnit4ClassRunner.java:79)
-	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:71)
-	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:49)
-	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:193)
-	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:52)
-	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:191)
-	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:42)
-	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:184)
-	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
-	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
-	at org.junit.runners.ParentRunner.run(ParentRunner.java:236)
-	at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:236)
-	at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:134)
-	at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:113)
-	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
-	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
-	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
-	at java.lang.reflect.Method.invoke(Method.java:597)
-	at org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray(ReflectionUtils.java:189)
-	at org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:165)
-	at org.apache.maven.surefire.booter.ProviderFactory.invokeProvider(ProviderFactory.java:85)
-	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:103)
-	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:74)
+     <blockquote>The following exception is thrown
+===========
+*org.apache.hadoop.HadoopIllegalArgumentException: Cannot close proxy - is not Closeable or does not provide closeable invocation handler class org.apache.hadoop.yarn.api.impl.pb.client.ClientRMProtocolPBClientImpl*
+	*at org.apache.hadoop.ipc.RPC.stopProxy(RPC.java:624)*
+	*at org.hadoop.yarn.client.YarnClientImpl.stop(YarnClientImpl.java:102)*
+	at org.apache.hadoop.yarn.applications.unmanagedamlauncher.UnmanagedAMLauncher.run(UnmanagedAMLauncher.java:336)
+	at org.apache.hadoop.yarn.applications.unmanagedamlauncher.TestUnmanagedAMLauncher.testDSShell(TestUnmanagedAMLauncher.java:156)
+	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
+	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
+	at java.lang.reflect.Method.invoke(Method.java:597)
+	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
+	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
+	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
+	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
+	at org.junit.runners.BlockJUnit4ClassRunner.runNotIgnored(BlockJUnit4ClassRunner.java:79)
+	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:71)
+	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:49)
+	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:193)
+	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:52)
+	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:191)
+	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:42)
+	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:184)
+	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
+	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:31)
+	at org.junit.runners.ParentRunner.run(ParentRunner.java:236)
+	at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:236)
+	at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:134)
+	at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:113)
+	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
+	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
+	at java.lang.reflect.Method.invoke(Method.java:597)
+	at org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray(ReflectionUtils.java:189)
+	at org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:165)
+	at org.apache.maven.surefire.booter.ProviderFactory.invokeProvider(ProviderFactory.java:85)
+	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:103)
+	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:74)
 ===========</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-75">YARN-75</a>.
      Major bug reported by Siddharth Seth and fixed by Siddharth Seth <br>
      <b>RMContainer should handle a RELEASE event while RUNNING</b><br>
-     <blockquote>An AppMaster can send a container release at any point. Currently this results in an exception, if this is done while the RM considers the container to be RUNNING.
+     <blockquote>An AppMaster can send a container release at any point. Currently this results in an exception, if this is done while the RM considers the container to be RUNNING.
 The event not being processed correctly also implies that these containers do not show up in the Completed Container List seen by the AM (AMRMProtocol). MR-3902 depends on this set being complete. </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-68">YARN-68</a>.
      Major bug reported by patrick white and fixed by Daryn Sharp (nodemanager)<br>
      <b>NodeManager will refuse to shutdown indefinitely due to container log aggregation</b><br>
-     <blockquote>The nodemanager is able to get into a state where containermanager.logaggregation.AppLogAggregatorImpl will apparently wait
-indefinitely for log aggregation to complete for an application, even if that application has abnormally terminated and is no longer present. 
-
-Observed behavior is that an attempt to stop the nodemanager daemon will return but have no effect, the nm log continually displays messages similar to this:
-
-[Thread-1]2012-08-21 17:44:07,581 INFO
-org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AppLogAggregatorImpl:
-Waiting for aggregation to complete for application_1345221477405_2733
-
-The only recovery we found to work was to 'kill -9' the nm process.
-
-What exactly causes the NM to enter this state is unclear but we do see this behavior reliably when the NM has run a task which failed, for example when debugging oozie distcp actions and having a distcp map task fail, the NM that was running the container will now enter this state where a shutdown on said NM will never complete, 'never' in this case was waiting for 2 hours before killing the nodemanager process.
+     <blockquote>The nodemanager is able to get into a state where containermanager.logaggregation.AppLogAggregatorImpl will apparently wait
+indefinitely for log aggregation to complete for an application, even if that application has abnormally terminated and is no longer present. 
+
+Observed behavior is that an attempt to stop the nodemanager daemon will return but have no effect, the nm log continually displays messages similar to this:
+
+[Thread-1]2012-08-21 17:44:07,581 INFO
+org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AppLogAggregatorImpl:
+Waiting for aggregation to complete for application_1345221477405_2733
+
+The only recovery we found to work was to 'kill -9' the nm process.
+
+What exactly causes the NM to enter this state is unclear but we do see this behavior reliably when the NM has run a task which failed, for example when debugging oozie distcp actions and having a distcp map task fail, the NM that was running the container will now enter this state where a shutdown on said NM will never complete, 'never' in this case was waiting for 2 hours before killing the nodemanager process.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-66">YARN-66</a>.
      Critical bug reported by Thomas Graves and fixed by Thomas Graves (nodemanager)<br>
      <b>aggregated logs permissions not set properly</b><br>
-     <blockquote>If the default file permissions are set to something restrictive - like 700, application logs get aggregated and created with those restrictive file permissions which doesn't allow the history server to serve them up.
-
-
-They need to be created with group readable similar to how log aggregation sets up the directory permissions.
+     <blockquote>If the default file permissions are set to something restrictive - like 700, application logs get aggregated and created with those restrictive file permissions which doesn't allow the history server to serve them up.
+
+
+They need to be created with group readable similar to how log aggregation sets up the directory permissions.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-63">YARN-63</a>.
      Major bug reported by Jason Lowe and fixed by Jason Lowe (resourcemanager)<br>
@@ -128,47 +128,47 @@ They need to be created with group readable similar to how log aggregation sets
 <li> <a href="https://issues.apache.org/jira/browse/YARN-42">YARN-42</a>.
      Major bug reported by Devaraj K and fixed by Devaraj K (nodemanager)<br>
      <b>Node Manager throws NPE on startup</b><br>
-     <blockquote>NM throws NPE on startup if it doesn't have persmission's on nm local dir's
-
-
-{code:xml}
-2012-05-14 16:32:13,468 FATAL org.apache.hadoop.yarn.server.nodemanager.NodeManager: Error starting NodeManager
-org.apache.hadoop.yarn.YarnException: Failed to initialize LocalizationService
-	at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.init(ResourceLocalizationService.java:202)
-	at org.apache.hadoop.yarn.service.CompositeService.init(CompositeService.java:58)
-	at org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.init(ContainerManagerImpl.java:183)
-	at org.apache.hadoop.yarn.service.CompositeService.init(CompositeService.java:58)
-	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.init(NodeManager.java:166)
-	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:268)
-	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:284)
-Caused by: java.io.IOException: mkdir of /mrv2/tmp/nm-local-dir/usercache failed
-	at org.apache.hadoop.fs.FileSystem.primitiveMkdir(FileSystem.java:907)
-	at org.apache.hadoop.fs.DelegateToFileSystem.mkdir(DelegateToFileSystem.java:143)
-	at org.apache.hadoop.fs.FilterFs.mkdir(FilterFs.java:189)
-	at org.apache.hadoop.fs.FileContext$4.next(FileContext.java:706)
-	at org.apache.hadoop.fs.FileContext$4.next(FileContext.java:703)
-	at org.apache.hadoop.fs.FileContext$FSLinkResolver.resolve(FileContext.java:2325)
-	at org.apache.hadoop.fs.FileContext.mkdir(FileContext.java:703)
-	at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.init(ResourceLocalizationService.java:188)
-	... 6 more
-2012-05-14 16:32:13,472 INFO org.apache.hadoop.yarn.service.CompositeService: Error stopping org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler
-java.lang.NullPointerException
-	at org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler.stop(NonAggregatingLogHandler.java:82)
-	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:99)
-	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:89)
-	at org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.stop(ContainerManagerImpl.java:266)
-	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:99)
-	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:89)
-	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.stop(NodeManager.java:182)
-	at org.apache.hadoop.yarn.service.CompositeService$CompositeServiceShutdownHook.run(CompositeService.java:122)
-	at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
-{code}
+     <blockquote>NM throws NPE on startup if it doesn't have persmission's on nm local dir's
+
+
+{code:xml}
+2012-05-14 16:32:13,468 FATAL org.apache.hadoop.yarn.server.nodemanager.NodeManager: Error starting NodeManager
+org.apache.hadoop.yarn.YarnException: Failed to initialize LocalizationService
+	at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.init(ResourceLocalizationService.java:202)
+	at org.apache.hadoop.yarn.service.CompositeService.init(CompositeService.java:58)
+	at org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.init(ContainerManagerImpl.java:183)
+	at org.apache.hadoop.yarn.service.CompositeService.init(CompositeService.java:58)
+	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.init(NodeManager.java:166)
+	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:268)
+	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:284)
+Caused by: java.io.IOException: mkdir of /mrv2/tmp/nm-local-dir/usercache failed
+	at org.apache.hadoop.fs.FileSystem.primitiveMkdir(FileSystem.java:907)
+	at org.apache.hadoop.fs.DelegateToFileSystem.mkdir(DelegateToFileSystem.java:143)
+	at org.apache.hadoop.fs.FilterFs.mkdir(FilterFs.java:189)
+	at org.apache.hadoop.fs.FileContext$4.next(FileContext.java:706)
+	at org.apache.hadoop.fs.FileContext$4.next(FileContext.java:703)
+	at org.apache.hadoop.fs.FileContext$FSLinkResolver.resolve(FileContext.java:2325)
+	at org.apache.hadoop.fs.FileContext.mkdir(FileContext.java:703)
+	at org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.init(ResourceLocalizationService.java:188)
+	... 6 more
+2012-05-14 16:32:13,472 INFO org.apache.hadoop.yarn.service.CompositeService: Error stopping org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler
+java.lang.NullPointerException
+	at org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler.stop(NonAggregatingLogHandler.java:82)
+	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:99)
+	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:89)
+	at org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.stop(ContainerManagerImpl.java:266)
+	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:99)
+	at org.apache.hadoop.yarn.service.CompositeService.stop(CompositeService.java:89)
+	at org.apache.hadoop.yarn.server.nodemanager.NodeManager.stop(NodeManager.java:182)
+	at org.apache.hadoop.yarn.service.CompositeService$CompositeServiceShutdownHook.run(CompositeService.java:122)
+	at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
+{code}
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-39">YARN-39</a>.
      Critical sub-task reported by Vinod Kumar Vavilapalli and fixed by Vinod Kumar Vavilapalli <br>
      <b>RM-NM secret-keys should be randomly generated and rolled every so often</b><br>
-     <blockquote> - RM should generate the master-key randomly
- - The master-key should roll every so often
+     <blockquote> - RM should generate the master-key randomly
+ - The master-key should roll every so often
  - NM should remember old expired keys so that already doled out container-requests can be satisfied.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-37">YARN-37</a>.
      Minor bug reported by Jason Lowe and fixed by Mayank Bansal (resourcemanager)<br>
@@ -177,42 +177,42 @@ java.lang.NullPointerException
 <li> <a href="https://issues.apache.org/jira/browse/YARN-36">YARN-36</a>.
      Blocker bug reported by Eli Collins and fixed by Radim Kolar <br>
      <b>branch-2.1.0-alpha doesn't build</b><br>
-     <blockquote>branch-2.1.0-alpha doesn't build due to the following. Per YARN-1 I updated the mvn version to be 2.1.0-SNAPSHOT, before I hit this issue it didn't compile due to the bogus version. 
-
-{noformat}
-hadoop-branch-2.1.0-alpha $ mvn compile
-[INFO] Scanning for projects...
-[ERROR] The build could not read 1 project -&gt; [Help 1]
-[ERROR]   
-[ERROR]   The project org.apache.hadoop:hadoop-yarn-project:2.1.0-SNAPSHOT (/home/eli/src/hadoop-branch-2.1.0-alpha/hadoop-yarn-project/pom.xml) has 1 error
-[ERROR]     'dependencies.dependency.version' for org.hsqldb:hsqldb:jar is missing. @ line 160, column 17
+     <blockquote>branch-2.1.0-alpha doesn't build due to the following. Per YARN-1 I updated the mvn version to be 2.1.0-SNAPSHOT, before I hit this issue it didn't compile due to the bogus version. 
+
+{noformat}
+hadoop-branch-2.1.0-alpha $ mvn compile
+[INFO] Scanning for projects...
+[ERROR] The build could not read 1 project -&gt; [Help 1]
+[ERROR]   
+[ERROR]   The project org.apache.hadoop:hadoop-yarn-project:2.1.0-SNAPSHOT (/home/eli/src/hadoop-branch-2.1.0-alpha/hadoop-yarn-project/pom.xml) has 1 error
+[ERROR]     'dependencies.dependency.version' for org.hsqldb:hsqldb:jar is missing. @ line 160, column 17
 {noformat}</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-31">YARN-31</a>.
      Major bug reported by Thomas Graves and fixed by Thomas Graves <br>
      <b>TestDelegationTokenRenewer fails on jdk7</b><br>
-     <blockquote>TestDelegationTokenRenewer fails when run with jdk7.  
-
+     <blockquote>TestDelegationTokenRenewer fails when run with jdk7.  
+
 With JDK7, test methods run in an undefined order. Here it is expecting that testDTRenewal runs first but it no longer is.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-29">YARN-29</a>.
      Major bug reported by Vinod Kumar Vavilapalli and fixed by Vinod Kumar Vavilapalli (client)<br>
      <b>Add a yarn-client module</b><br>
-     <blockquote>I see that we are duplicating (some) code for talking to RM via client API. In this light, a yarn-client module will be useful so that clients of all frameworks can use/extend it.
-
+     <blockquote>I see that we are duplicating (some) code for talking to RM via client API. In this light, a yarn-client module will be useful so that clients of all frameworks can use/extend it.
+
 And that same module can be the destination for all the YARN's command line tools.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-27">YARN-27</a>.
      Major bug reported by Ramya Sunil and fixed by Arun C Murthy <br>
      <b>Failed refreshQueues due to misconfiguration prevents further refreshing of queues</b><br>
-     <blockquote>Stumbled upon this problem while refreshing queues with incorrect configuration. The exact scenario was:
-1. Added a new queue "newQueue" without defining its capacity.
-2. "bin/mapred queue -refreshQueues" fails correctly with "Illegal capacity of -1 for queue root.newQueue"
-3. However, after defining the capacity of "newQueue" followed by a second "bin/mapred queue -refreshQueues" throws "org.apache.hadoop.metrics2.MetricsException: Metrics source QueueMetrics,q0=root,q1=newQueue already exists!" Also see Hadoop:name=QueueMetrics,q0=root,q1=newQueue,service=ResourceManager metrics being available even though the queue was not added.
-
+     <blockquote>Stumbled upon this problem while refreshing queues with incorrect configuration. The exact scenario was:
+1. Added a new queue "newQueue" without defining its capacity.
+2. "bin/mapred queue -refreshQueues" fails correctly with "Illegal capacity of -1 for queue root.newQueue"
+3. However, after defining the capacity of "newQueue" followed by a second "bin/mapred queue -refreshQueues" throws "org.apache.hadoop.metrics2.MetricsException: Metrics source QueueMetrics,q0=root,q1=newQueue already exists!" Also see Hadoop:name=QueueMetrics,q0=root,q1=newQueue,service=ResourceManager metrics being available even though the queue was not added.
+
 The expected behavior would be to refresh the queues correctly and allow addition of "newQueue". </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-25">YARN-25</a>.
      Major bug reported by Thomas Graves and fixed by Robert Joseph Evans <br>
      <b>remove old aggregated logs</b><br>
-     <blockquote>Currently the aggregated user logs under NM_REMOTE_APP_LOG_DIR are never removed.  We should have mechanism to remove them after certain period.
-
+     <blockquote>Currently the aggregated user logs under NM_REMOTE_APP_LOG_DIR are never removed.  We should have mechanism to remove them after certain period.
+
 It might make sense for job history server to remove them.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-22">YARN-22</a>.
      Minor bug reported by Eli Collins and fixed by Mayank Bansal <br>
@@ -221,29 +221,29 @@ It might make sense for job history server to remove them.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-15">YARN-15</a>.
      Critical bug reported by Alejandro Abdelnur and fixed by Arun C Murthy (nodemanager)<br>
      <b>YarnConfiguration DEFAULT_YARN_APPLICATION_CLASSPATH should be updated</b><br>
-     <blockquote>
-{code}
-  /**
-   * Default CLASSPATH for YARN applications. A comma-separated list of
-   * CLASSPATH entries
-   */
-  public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
-      "$HADOOP_CONF_DIR", "$HADOOP_COMMON_HOME/share/hadoop/common/*",
-      "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*",
-      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*",
-      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*",
-      "$YARN_HOME/share/hadoop/mapreduce/*",
-      "$YARN_HOME/share/hadoop/mapreduce/lib/*"};
-{code}
-
+     <blockquote>
+{code}
+  /**
+   * Default CLASSPATH for YARN applications. A comma-separated list of
+   * CLASSPATH entries
+   */
+  public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
+      "$HADOOP_CONF_DIR", "$HADOOP_COMMON_HOME/share/hadoop/common/*",
+      "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*",
+      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*",
+      "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*",
+      "$YARN_HOME/share/hadoop/mapreduce/*",
+      "$YARN_HOME/share/hadoop/mapreduce/lib/*"};
+{code}
+
 It should have {{share/yarn/}} and MR should add the {{share/mapreduce/}} (another JIRA?)</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-14">YARN-14</a>.
      Major bug reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
      <b>Symlinks to peer distributed cache files no longer work</b><br>
-     <blockquote>Trying to create a symlink to another file that is specified for the distributed cache will fail to create the link.  For example:
-
-hadoop jar ... -files "x,y,x#z"
-
+     <blockquote>Trying to create a symlink to another file that is specified for the distributed cache will fail to create the link.  For example:
+
+hadoop jar ... -files "x,y,x#z"
+
 will localize the files x and y as x and y, but the z symlink for x will not be created.  This is a regression from 1.x behavior.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-13">YARN-13</a>.
      Critical bug reported by Todd Lipcon and fixed by  <br>
@@ -252,13 +252,13 @@ will localize the files x and y as x and y, but the z symlink for x will not be
 <li> <a href="https://issues.apache.org/jira/browse/YARN-12">YARN-12</a>.
      Major bug reported by Junping Du and fixed by Junping Du (scheduler)<br>
      <b>Several Findbugs issues with new FairScheduler in YARN</b><br>
-     <blockquote>The good feature of FairScheduler is added recently to YARN. As recently PreCommit test from MAPREDUCE-4309, there are several bugs found by Findbugs related to FairScheduler:
-org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog.shutdown() might ignore java.lang.Exception
-Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog.logDisabled; locked 50% of time
-Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager.queueMaxAppsDefault; locked 50% of time
-Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager.userMaxAppsDefault; locked 50% of time
-The details are in:https://builds.apache.org/job/PreCommit-MAPREDUCE-Build/2612//artifact/trunk/patchprocess/newPatchFindbugsWarningshadoop-yarn-server-resourcemanager.html#DE_MIGHT_IGNORE 
-
+     <blockquote>The good feature of FairScheduler is added recently to YARN. As recently PreCommit test from MAPREDUCE-4309, there are several bugs found by Findbugs related to FairScheduler:
+org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog.shutdown() might ignore java.lang.Exception
+Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog.logDisabled; locked 50% of time
+Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager.queueMaxAppsDefault; locked 50% of time
+Inconsistent synchronization of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager.userMaxAppsDefault; locked 50% of time
+The details are in:https://builds.apache.org/job/PreCommit-MAPREDUCE-Build/2612//artifact/trunk/patchprocess/newPatchFindbugsWarningshadoop-yarn-server-resourcemanager.html#DE_MIGHT_IGNORE 
+
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/YARN-10">YARN-10</a>.
      Major improvement reported by Arun C Murthy and fixed by Hitesh Shah <br>
@@ -991,18 +991,18 @@ The details are in:https://builds.apache.org/job/PreCommit-MAPREDUCE-Build/2612/
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3812">MAPREDUCE-3812</a>.
      Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Harsh J (mrv2 , performance)<br>
      <b>Lower default allocation sizes, fix allocation configurations and document them</b><br>
-     <blockquote>Removes two sets of previously available config properties:
-
-1. ( yarn.scheduler.fifo.minimum-allocation-mb and yarn.scheduler.fifo.maximum-allocation-mb ) and,
-2. ( yarn.scheduler.capacity.minimum-allocation-mb and yarn.scheduler.capacity.maximum-allocation-mb )
-
-In favor of two new, generically named properties:
-
-1. yarn.scheduler.minimum-allocation-mb - This acts as the floor value of memory resource requests for containers.
-2. yarn.scheduler.maximum-allocation-mb - This acts as the ceiling value of memory resource requests for containers.
-
-Both these properties need to be set at the ResourceManager (RM) to take effect, as the RM is where the scheduler resides.
-
+     <blockquote>Removes two sets of previously available config properties:
+
+1. ( yarn.scheduler.fifo.minimum-allocation-mb and yarn.scheduler.fifo.maximum-allocation-mb ) and,
+2. ( yarn.scheduler.capacity.minimum-allocation-mb and yarn.scheduler.capacity.maximum-allocation-mb )
+
+In favor of two new, generically named properties:
+
+1. yarn.scheduler.minimum-allocation-mb - This acts as the floor value of memory resource requests for containers.
+2. yarn.scheduler.maximum-allocation-mb - This acts as the ceiling value of memory resource requests for containers.
+
+Both these properties need to be set at the ResourceManager (RM) to take effect, as the RM is where the scheduler resides.
+
 Also changes the default minimum and maximums to 128 MB and 10 GB respectively.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3782">MAPREDUCE-3782</a>.
      Critical bug reported by Arpit Gupta and fixed by Jason Lowe (mrv2)<br>
@@ -1043,8 +1043,8 @@ Also changes the default minimum and maximums to 128 MB and 10 GB respectively.<
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3543">MAPREDUCE-3543</a>.
      Critical bug reported by Mahadev konar and fixed by Thomas Graves (mrv2)<br>
      <b>Mavenize Gridmix.</b><br>
-     <blockquote>Note that to apply this you should first run the script - ./MAPREDUCE-3543v3.sh svn, then apply the patch.
-
+     <blockquote>Note that to apply this you should first run the script - ./MAPREDUCE-3543v3.sh svn, then apply the patch.
+
 If this is merged to more then trunk, the version inside of hadoop-tools/hadoop-gridmix/pom.xml will need to be udpated accordingly.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3506">MAPREDUCE-3506</a>.
      Minor bug reported by Ratandeep Ratti and fixed by Jason Lowe (client , mrv2)<br>
@@ -1613,10 +1613,10 @@ If this is merged to more then trunk, the version inside of hadoop-tools/hadoop-
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-3475">HDFS-3475</a>.
      Trivial improvement reported by Harsh J and fixed by Harsh J <br>
      <b>Make the replication and invalidation rates configurable</b><br>
-     <blockquote>This change adds two new configuration parameters. 
-# {{dfs.namenode.invalidate.work.pct.per.iteration}} for controlling deletion rate of blocks. 
-# {{dfs.namenode.replication.work.multiplier.per.iteration}} for controlling replication rate. This in turn allows controlling the time it takes for decommissioning. 
-
+     <blockquote>This change adds two new configuration parameters. 
+# {{dfs.namenode.invalidate.work.pct.per.iteration}} for controlling deletion rate of blocks. 
+# {{dfs.namenode.replication.work.multiplier.per.iteration}} for controlling replication rate. This in turn allows controlling the time it takes for decommissioning. 
+
 Please see hdfs-default.xml for detailed description.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-3474">HDFS-3474</a>.
      Major sub-task reported by Ivan Kelly and fixed by Ivan Kelly <br>
@@ -4769,8 +4769,8 @@ These release notes include new developer and user-facing incompatibilities, fea
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3720">MAPREDUCE-3720</a>.
      Major bug reported by Vinod Kumar Vavilapalli and fixed by Vinod Kumar Vavilapalli (client , mrv2)<br>
      <b>Command line listJobs should not visit each AM</b><br>
-     <blockquote>Changed bin/mapred job -list to not print job-specific information not available at RM.
-
+     <blockquote>Changed bin/mapred job -list to not print job-specific information not available at RM.
+
 Very minor incompatibility in cmd-line output, inevitable due to MRv2 architecture.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3718">MAPREDUCE-3718</a>.
      Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Hitesh Shah (mrv2 , performance)<br>
@@ -4819,8 +4819,8 @@ Very minor incompatibility in cmd-line output, inevitable due to MRv2 architectu
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3703">MAPREDUCE-3703</a>.
      Critical bug reported by Eric Payne and fixed by Eric Payne (mrv2 , resourcemanager)<br>
      <b>ResourceManager should provide node lists in JMX output</b><br>
-     <blockquote>New JMX Bean in ResourceManager to provide list of live node managers:
-
+     <blockquote>New JMX Bean in ResourceManager to provide list of live node managers:
+
 Hadoop:service=ResourceManager,name=RMNMInfo LiveNodeManagers</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3702">MAPREDUCE-3702</a>.
      Critical bug reported by Thomas Graves and fixed by Thomas Graves (mrv2)<br>
@@ -5037,12 +5037,12 @@ Hadoop:service=ResourceManager,name=RMNMInfo LiveNodeManagers</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3549">MAPREDUCE-3549</a>.
      Blocker bug reported by Thomas Graves and fixed by Thomas Graves (mrv2)<br>
      <b>write api documentation for web service apis for RM, NM, mapreduce app master, and job history server</b><br>
-     <blockquote>new files added: A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm
-A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
-A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
-A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
-A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
-
+     <blockquote>new files added: A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm
+A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
+A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
+A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
+A      hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
+
 The hadoop-project/src/site/site.xml is split into separate patch.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3548">MAPREDUCE-3548</a>.
      Critical sub-task reported by Thomas Graves and fixed by Thomas Graves (mrv2)<br>
@@ -5471,7 +5471,7 @@ The hadoop-project/src/site/site.xml is split into separate patch.</blockquote><
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3297">MAPREDUCE-3297</a>.
      Major task reported by Siddharth Seth and fixed by Siddharth Seth (mrv2)<br>
      <b>Move Log Related components from yarn-server-nodemanager to yarn-common</b><br>
-     <blockquote>Moved log related components into yarn-common so that HistoryServer and clients can use them without depending on the yarn-server-nodemanager module.
+     <blockquote>Moved log related components into yarn-common so that HistoryServer and clients can use them without depending on the yarn-server-nodemanager module.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3291">MAPREDUCE-3291</a>.
      Blocker bug reported by Ramya Sunil and fixed by Robert Joseph Evans (mrv2)<br>
@@ -5504,17 +5504,17 @@ The hadoop-project/src/site/site.xml is split into separate patch.</blockquote><
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3219">MAPREDUCE-3219</a>.
      Minor sub-task reported by Hitesh Shah and fixed by Hitesh Shah (mrv2 , test)<br>
      <b>ant test TestDelegationToken failing on trunk</b><br>
-     <blockquote>Reenabled and fixed bugs in the failing test TestDelegationToken.
+     <blockquote>Reenabled and fixed bugs in the failing test TestDelegationToken.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3217">MAPREDUCE-3217</a>.
      Minor sub-task reported by Hitesh Shah and fixed by Devaraj K (mrv2 , test)<br>
      <b>ant test TestAuditLogger fails on trunk</b><br>
-     <blockquote>Reenabled and fixed bugs in the failing ant test TestAuditLogger.
+     <blockquote>Reenabled and fixed bugs in the failing ant test TestAuditLogger.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3215">MAPREDUCE-3215</a>.
      Minor sub-task reported by Hitesh Shah and fixed by Hitesh Shah (mrv2)<br>
      <b>org.apache.hadoop.mapreduce.TestNoJobSetupCleanup failing on trunk</b><br>
-     <blockquote>Reneabled and fixed bugs in the failing test TestNoJobSetupCleanup.
+     <blockquote>Reneabled and fixed bugs in the failing test TestNoJobSetupCleanup.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3194">MAPREDUCE-3194</a>.
      Major bug reported by Siddharth Seth and fixed by Jason Lowe (mrv2)<br>
@@ -5875,12 +5875,12 @@ The hadoop-project/src/site/site.xml is split into separate patch.</blockquote><
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-2246">HDFS-2246</a>.
      Major improvement reported by Sanjay Radia and fixed by Jitendra Nath Pandey <br>
      <b>Shortcut a local client reads to a Datanodes files directly</b><br>
-     <blockquote>1. New configurations
-a. dfs.block.local-path-access.user is the key in datanode configuration to specify the user allowed to do short circuit read.
-b. dfs.client.read.shortcircuit is the key to enable short circuit read at the client side configuration.
-c. dfs.client.read.shortcircuit.skip.checksum is the key to bypass checksum check at the client side.
-2. By default none of the above are enabled and short circuit read will not kick in.
-3. If security is on, the feature can be used only for user that has kerberos credentials at the client, therefore map reduce tasks cannot benefit from it in general.
+     <blockquote>1. New configurations
+a. dfs.block.local-path-access.user is the key in datanode configuration to specify the user allowed to do short circuit read.
+b. dfs.client.read.shortcircuit is the key to enable short circuit read at the client side configuration.
+c. dfs.client.read.shortcircuit.skip.checksum is the key to bypass checksum check at the client side.
+2. By default none of the above are enabled and short circuit read will not kick in.
+3. If security is on, the feature can be used only for user that has kerberos credentials at the client, therefore map reduce tasks cannot benefit from it in general.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-2178">HDFS-2178</a>.
      Major improvement reported by Alejandro Abdelnur and fixed by Alejandro Abdelnur <br>
@@ -6161,7 +6161,7 @@ c. dfs.client.read.shortcircuit.skip.checksum is the key to bypass checksum chec
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7802">HADOOP-7802</a>.
      Major bug reported by Bruno Mah&#233; and fixed by Bruno Mah&#233; <br>
      <b>Hadoop scripts unconditionally source "$bin"/../libexec/hadoop-config.sh.</b><br>
-     <blockquote>Here is a patch to enable this behavior
+     <blockquote>Here is a patch to enable this behavior
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7801">HADOOP-7801</a>.
      Major bug reported by Bruno Mah&#233; and fixed by Bruno Mah&#233; (build)<br>
@@ -6486,9 +6486,9 @@ These release notes include new developer and user-facing incompatibilities, fea
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3186">MAPREDUCE-3186</a>.
      Blocker bug reported by Ramgopal N and fixed by Eric Payne (mrv2)<br>
      <b>User jobs are getting hanged if the Resource manager process goes down and comes up while job is getting executed.</b><br>
-     <blockquote>New Yarn configuration property:
-
-Name: yarn.app.mapreduce.am.scheduler.connection.retries
+     <blockquote>New Yarn configuration property:
+
+Name: yarn.app.mapreduce.am.scheduler.connection.retries
 Description: Number of times AM should retry to contact RM if connection is lost.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3185">MAPREDUCE-3185</a>.
      Critical bug reported by Mahadev konar and fixed by Jonathan Eagles (mrv2)<br>
@@ -6641,7 +6641,7 @@ Description: Number of times AM should retry to contact RM if connection is lost
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3112">MAPREDUCE-3112</a>.
      Major bug reported by Eric Yang and fixed by Eric Yang (contrib/streaming)<br>
      <b>Calling hadoop cli inside mapreduce job leads to errors</b><br>
-     <blockquote>Removed inheritance of certain server environment variables (HADOOP_OPTS and HADOOP_ROOT_LOGGER) in task attempt process.
+     <blockquote>Removed inheritance of certain server environment variables (HADOOP_OPTS and HADOOP_ROOT_LOGGER) in task attempt process.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-3110">MAPREDUCE-3110</a>.
      Major bug reported by Devaraj K and fixed by Vinod Kumar Vavilapalli (mrv2 , test)<br>
@@ -7114,16 +7114,16 @@ Description: Number of times AM should retry to contact RM if connection is lost
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-2858">MAPREDUCE-2858</a>.
      Blocker sub-task reported by Luke Lu and fixed by Robert Joseph Evans (applicationmaster , mrv2 , security)<br>
      <b>MRv2 WebApp Security</b><br>
-     <blockquote>A new server has been added to yarn.  It is a web proxy that sits in front of the AM web UI.  The server is controlled by the yarn.web-proxy.address config.  If that config is set, and it points to an address that is different then the RM web interface then a separate proxy server needs to be launched.
-
-This can be done by running 
-
-yarn-daemon.sh start proxyserver
-
-If a separate proxy server is needed other configs also may need to be set, if security is enabled.
-yarn.web-proxy.principal
-yarn.web-proxy.keytab
-
+     <blockquote>A new server has been added to yarn.  It is a web proxy that sits in front of the AM web UI.  The server is controlled by the yarn.web-proxy.address config.  If that config is set, and it points to an address that is different then the RM web interface then a separate proxy server needs to be launched.
+
+This can be done by running 
+
+yarn-daemon.sh start proxyserver
+
+If a separate proxy server is needed other configs also may need to be set, if security is enabled.
+yarn.web-proxy.principal
+yarn.web-proxy.keytab
+
 The proxy server is stateless and should be able to support a VIP or other load balancing sitting in front of multiple instances of this server.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-2854">MAPREDUCE-2854</a>.
      Major bug reported by Thomas Graves and fixed by Thomas Graves <br>
@@ -8061,12 +8061,12 @@ mapreduce.reduce.shuffle.catch.exception.message.regex</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-2037">MAPREDUCE-2037</a>.
      Major new feature reported by Dick King and fixed by Dick King <br>
      <b>Capturing interim progress times, CPU usage, and memory usage, when tasks reach certain progress thresholds</b><br>
-     <blockquote>Capture intermediate task resource consumption information:
-* Time taken so far
-* CPU load [either at the time the data are taken, or exponentially smoothed]
-* Memory load [also either at the time the data are taken, or exponentially smoothed]
-
-This would be taken at intervals that depend on the task progress plateaus. For example, reducers have three progress ranges - [0-1/3], (1/3-2/3], and (2/3-3/3] - where fundamentally different activities happen. Mappers have different boundaries that are not symmetrically placed [0-9/10], (9/10-1]. Data capture boundaries should coincide with activity boundaries. For the state information capture [CPU and memory] we should average over the covered interval.
+     <blockquote>Capture intermediate task resource consumption information:
+* Time taken so far
+* CPU load [either at the time the data are taken, or exponentially smoothed]
+* Memory load [also either at the time the data are taken, or exponentially smoothed]
+
+This would be taken at intervals that depend on the task progress plateaus. For example, reducers have three progress ranges - [0-1/3], (1/3-2/3], and (2/3-3/3] - where fundamentally different activities happen. Mappers have different boundaries that are not symmetrically placed [0-9/10], (9/10-1]. Data capture boundaries should coincide with activity boundaries. For the state information capture [CPU and memory] we should average over the covered interval.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-2033">MAPREDUCE-2033</a>.
      Major task reported by Vinay Kumar Thota and fixed by Vinay Kumar Thota (contrib/gridmix)<br>
@@ -8175,24 +8175,24 @@ This would be taken at intervals that depend on the task progress plateaus. For
 <li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-279">MAPREDUCE-279</a>.
      Major improvement reported by Arun C Murthy and fixed by  (mrv2)<br>
      <b>Map-Reduce 2.0</b><br>
-     <blockquote>MapReduce has undergone a complete re-haul in hadoop-0.23 and we now have, what we call, MapReduce 2.0 (MRv2).
-
-The fundamental idea of MRv2 is to split up the two major functionalities of the JobTracker, resource management and job scheduling/monitoring, into separate daemons. The idea is to have a global ResourceManager (RM) and per-application ApplicationMaster (AM).  An application is either a single job in the classical sense of Map-Reduce jobs or a DAG of jobs. The ResourceManager and per-node slave, the NodeManager (NM), form the data-computation framework. The ResourceManager is the ultimate authority that arbitrates resources among all the applications in the system. The per-application ApplicationMaster is, in effect, a framework specific library and is tasked with negotiating resources from the ResourceManager and working with the NodeManager(s) to execute and monitor the tasks.
-
-The ResourceManager has two main components:
-* Scheduler (S)
-* ApplicationsManager (ASM)
-
-The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees on restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based the resource requirements of the applications; it does so based on the abstract notion of a Resource Container which incorporates elements such as memory, cpu, disk, network etc. 
-
-The Scheduler has a pluggable policy plug-in, which is responsible for partitioning the cluster resources among the various queues, applications etc. The current Map-Reduce schedulers such as the CapacityScheduler and the FairScheduler would be some examples of the plug-in.
-
-The CapacityScheduler supports hierarchical queues to allow for more predictable sharing of cluster resources.
-The ApplicationsManager is responsible for accepting job-submissions, negotiating the first container for executing the application specific ApplicationMaster and provides the service for restarting the ApplicationMaster container on failure.
-
-The NodeManager is the per-machine framework agent who is responsible for launching the applications' containers, monitoring their resource usage (cpu, memory, disk, network) and reporting the same to the Scheduler.
-
-The per-application ApplicationMaster has the responsibility of negotiating appropriate resource containers from the Scheduler, tracking their status and monitoring for progress.
+     <blockquote>MapReduce has undergone a complete re-haul in hadoop-0.23 and we now have, what we call, MapReduce 2.0 (MRv2).
+
+The fundamental idea of MRv2 is to split up the two major functionalities of the JobTracker, resource management and job scheduling/monitoring, into separate daemons. The idea is to have a global ResourceManager (RM) and per-application ApplicationMaster (AM).  An application is either a single job in the classical sense of Map-Reduce jobs or a DAG of jobs. The ResourceManager and per-node slave, the NodeManager (NM), form the data-computation framework. The ResourceManager is the ultimate authority that arbitrates resources among all the applications in the system. The per-application ApplicationMaster is, in effect, a framework specific library and is tasked with negotiating resources from the ResourceManager and working with the NodeManager(s) to execute and monitor the tasks.
+
+The ResourceManager has two main components:
+* Scheduler (S)
+* ApplicationsManager (ASM)
+
+The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees on restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based the resource requirements of the applications; it does so based on the abstract notion of a Resource Container which incorporates elements such as memory, cpu, disk, network etc. 
+
+The Scheduler has a pluggable policy plug-in, which is responsible for partitioning the cluster resources among the various queues, applications etc. The current Map-Reduce schedulers such as the CapacityScheduler and the FairScheduler would be some examples of the plug-in.
+
+The CapacityScheduler supports hierarchical queues to allow for more predictable sharing of cluster resources.
+The ApplicationsManager is responsible for accepting job-submissions, negotiating the first container for executing the application specific ApplicationMaster and provides the service for restarting the ApplicationMaster container on failure.
+
+The NodeManager is the per-machine framework agent who is responsible for launching the applications' containers, monitoring their resource usage (cpu, memory, disk, network) and reporting the same to the Scheduler.
+
+The per-application ApplicationMaster has the responsibility of negotiating appropriate resource containers from the Scheduler, tracking their status and monitoring for progress.
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-2540">HDFS-2540</a>.
      Major sub-task reported by Tsz Wo (Nicholas), SZE and fixed by Tsz Wo (Nicholas), SZE <br>
@@ -8253,10 +8253,10 @@ The per-application ApplicationMaster has the responsibility of negotiating appr
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-2465">HDFS-2465</a>.
      Major improvement reported by Todd Lipcon and fixed by Todd Lipcon (data-node , performance)<br>
      <b>Add HDFS support for fadvise readahead and drop-behind</b><br>
-     <blockquote>HDFS now has the ability to use posix_fadvise and sync_data_range syscalls to manage the OS buffer cache. This support is currently considered experimental, and may be enabled by configuring the following keys:
-dfs.datanode.drop.cache.behind.writes - set to true to drop data out of the buffer cache after writing
-dfs.datanode.drop.cache.behind.reads - set to true to drop data out of the buffer cache when performing sequential reads
-dfs.datanode.sync.behind.writes - set to true to trigger dirty page writeback immediately after writing data
+     <blockquote>HDFS now has the ability to use posix_fadvise and sync_data_range syscalls to manage the OS buffer cache. This support is currently considered experimental, and may be enabled by configuring the following keys:
+dfs.datanode.drop.cache.behind.writes - set to true to drop data out of the buffer cache after writing
+dfs.datanode.drop.cache.behind.reads - set to true to drop data out of the buffer cache when performing sequential reads
+dfs.datanode.sync.behind.writes - set to true to trigger dirty page writeback immediately after writing data
 dfs.datanode.readahead.bytes - set to a non-zero value to trigger readahead for sequential reads</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-2453">HDFS-2453</a>.
      Major sub-task reported by Arpit Gupta and fixed by Tsz Wo (Nicholas), SZE (webhdfs)<br>
@@ -9331,7 +9331,7 @@ This is an incompatible change in 0.23.  The versions of ClientProtocol and Data
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1594">HDFS-1594</a>.
      Major bug reported by Devaraj K and fixed by Aaron T. Myers (name-node)<br>
      <b>When the disk becomes full Namenode is getting shutdown and not able to recover</b><br>
-     <blockquote>Implemented a daemon thread to monitor the disk usage for periodically and if the disk usage reaches the threshold value, put the name node into Safe mode so that no modification to file system will occur. Once the disk usage reaches below the threshold, name node will be put out of the safe mode. Here threshold value and interval to check the disk usage are configurable. 
+     <blockquote>Implemented a daemon thread to monitor the disk usage for periodically and if the disk usage reaches the threshold value, put the name node into Safe mode so that no modification to file system will occur. Once the disk usage reaches below the threshold, name node will be put out of the safe mode. Here threshold value and interval to check the disk usage are configurable. 
 </blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1592">HDFS-1592</a>.
      Major bug reported by Bharath Mundlapudi and fixed by Bharath Mundlapudi <br>
@@ -9376,9 +9376,9 @@ This is an incompatible change in 0.23.  The versions of ClientProtocol and Data
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1547">HDFS-1547</a>.
      Major improvement reported by Suresh Srinivas and fixed by Suresh Srinivas (name-node)<br>
      <b>Improve decommission mechanism</b><br>
-     <blockquote>Summary of changes to the decommissioning process:
-# After nodes are decommissioned, they are not shutdown. The decommissioned nodes are not used for writes. For reads, the decommissioned nodes are given as the last location to read from.
-# Number of live and dead decommissioned nodes are displayed in the namenode webUI.
+     <blockquote>Summary of changes to the decommissioning process:
+# After nodes are decommissioned, they are not shutdown. The decommissioned nodes are not used for writes. For reads, the decommissioned nodes are given as the last location to read from.
+# Number of live and dead decommissioned nodes are displayed in the namenode webUI.
 # Decommissioned nodes free capacity is not count towards the the cluster free capacity.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1541">HDFS-1541</a>.
      Major sub-task reported by Hairong Kuang and fixed by Hairong Kuang (name-node)<br>
@@ -9491,10 +9491,10 @@ This is an incompatible change in 0.23.  The versions of ClientProtocol and Data
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1448">HDFS-1448</a>.
      Major new feature reported by Erik Steffl and fixed by Erik Steffl (tools)<br>
      <b>Create multi-format parser for edits logs file, support binary and XML formats initially</b><br>
-     <blockquote>Offline edits viewer feature adds oev tool to hdfs script. Oev makes it possible to convert edits logs to/from native binary and XML formats. It uses the same framework as Offline image viewer.
-
-Example usage:
-
+     <blockquote>Offline edits viewer feature adds oev tool to hdfs script. Oev makes it possible to convert edits logs to/from native binary and XML formats. It uses the same framework as Offline image viewer.
+
+Example usage:
+
 $HADOOP_HOME/bin/hdfs oev -i edits -o output.xml</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HDFS-1445">HDFS-1445</a>.
      Major sub-task reported by Matt Foley and fixed by Matt Foley (data-node)<br>
@@ -9762,7 +9762,7 @@ This change requires an upgrade at deployment.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7681">HADOOP-7681</a>.
      Minor bug reported by Arpit Gupta and fixed by Arpit Gupta (conf)<br>
      <b>log4j.properties is missing properties for security audit and hdfs audit should be changed to info</b><br>
-     <blockquote>HADOOP-7681. Fixed security and hdfs audit log4j properties
+     <blockquote>HADOOP-7681. Fixed security and hdfs audit log4j properties
 (Arpit Gupta via Eric Yang)</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7671">HADOOP-7671</a>.
      Major bug reported by Ravi Prakash and fixed by Ravi Prakash <br>
@@ -10363,8 +10363,8 @@ This change requires an upgrade at deployment.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7227">HADOOP-7227</a>.
      Major improvement reported by Jitendra Nath Pandey and fixed by Jitendra Nath Pandey (ipc)<br>
      <b>Remove protocol version check at proxy creation in Hadoop RPC.</b><br>
-     <blockquote>1. Protocol version check is removed from proxy creation, instead version check is performed at server in every rpc call.
-2. This change is backward incompatible because format of the rpc messages is changed to include client version, client method hash and rpc version.
+     <blockquote>1. Protocol version check is removed from proxy creation, instead version check is performed at server in every rpc call.
+2. This change is backward incompatible because format of the rpc messages is changed to include client version, client method hash and rpc version.
 3. rpc version is introduced which should change when the format of rpc messages is changed.</blockquote></li>
 <li> <a href="https://issues.apache.org/jira/browse/HADOOP-7223">HADOOP-7223</a>.
      Major bug reported by Suresh Srinivas and fixed by Suresh Srinivas (fs)<br>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -24,6 +24,7 @@ import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -1391,11 +1392,11 @@ public abstract class FileSystem extends Configured implements Closeable {
   }
 
   final private static PathFilter DEFAULT_FILTER = new PathFilter() {
-      @Override
-      public boolean accept(Path file) {
-        return true;
-      }     
-    };
+    @Override
+    public boolean accept(Path file) {
+      return true;
+    }
+  };
     
   /**
    * List the statuses of the files/directories in the given path if the path is
@@ -1559,136 +1560,128 @@ public abstract class FileSystem extends Configured implements Closeable {
   }
   
   /**
-   * Return an array of FileStatus objects whose path names match pathPattern
-   * and is accepted by the user-supplied path filter. Results are sorted by
-   * their path names.
-   * Return null if pathPattern has no glob and the path does not exist.
-   * Return an empty array if pathPattern has a glob and no path matches it. 
+   * Return an array of FileStatus objects whose path names match
+   * {@code pathPattern} and is accepted by the user-supplied path filter.
+   * Results are sorted by their path names.
    * 
-   * @param pathPattern
-   *          a regular expression specifying the path pattern
-   * @param filter
-   *          a user-supplied path filter
-   * @return an array of FileStatus objects
+   * @param pathPattern a regular expression specifying the path pattern
+   * @param filter a user-supplied path filter
+   * @return null if {@code pathPattern} has no glob and the path does not exist
+   *         an empty array if {@code pathPattern} has a glob and no path
+   *         matches it else an array of {@link FileStatus} objects matching the
+   *         pattern
    * @throws IOException if any I/O error occurs when fetching file status
    */
   public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
       throws IOException {
     String filename = pathPattern.toUri().getPath();
+    List<FileStatus> allMatches = null;
+    
     List<String> filePatterns = GlobExpander.expand(filename);
-    if (filePatterns.size() == 1) {
-      return globStatusInternal(pathPattern, filter);
-    } else {
-      List<FileStatus> results = new ArrayList<FileStatus>();
-      for (String filePattern : filePatterns) {
-        FileStatus[] files = globStatusInternal(new Path(filePattern), filter);
-        for (FileStatus file : files) {
-          results.add(file);
+    for (String filePattern : filePatterns) {
+      Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern);
+      List<FileStatus> matches = globStatusInternal(path, filter);
+      if (matches != null) {
+        if (allMatches == null) {
+          allMatches = matches;
+        } else {
+          allMatches.addAll(matches);
         }
       }
-      return results.toArray(new FileStatus[results.size()]);
     }
+    
+    FileStatus[] results = null;
+    if (allMatches != null) {
+      results = allMatches.toArray(new FileStatus[allMatches.size()]);
+    } else if (filePatterns.size() > 1) {
+      // no matches with multiple expansions is a non-matching glob 
+      results = new FileStatus[0];
+    }
+    return results;
   }
 
-  private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter)
-      throws IOException {
-    Path[] parents = new Path[1];
+  // sort gripes because FileStatus Comparable isn't parameterized...
+  @SuppressWarnings("unchecked") 
+  private List<FileStatus> globStatusInternal(Path pathPattern,
+      PathFilter filter) throws IOException {
+    boolean patternHasGlob = false;       // pathPattern has any globs
+    List<FileStatus> matches = new ArrayList<FileStatus>();
+
+    // determine starting point
     int level = 0;
-    String filename = pathPattern.toUri().getPath();
+    String baseDir = Path.CUR_DIR;
+    if (pathPattern.isAbsolute()) {
+      level = 1; // need to skip empty item at beginning of split list
+      baseDir = Path.SEPARATOR;
+    }
     
-    // path has only zero component
-    if (filename.isEmpty() || Path.SEPARATOR.equals(filename)) {
-      return getFileStatus(new Path[]{pathPattern});
+    // parse components and determine if it's a glob
+    String[] components = null;
+    GlobFilter[] filters = null;
+    String filename = pathPattern.toUri().getPath();
+    if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) {
+      components = filename.split(Path.SEPARATOR);
+      filters = new GlobFilter[components.length];
+      for (int i=level; i < components.length; i++) {
+        filters[i] = new GlobFilter(components[i]);
+        patternHasGlob |= filters[i].hasPattern();
+      }
+      if (!patternHasGlob) {
+        baseDir = unquotePathComponent(filename);
+        components = null; // short through to filter check
+      }
     }
-
-    // path has at least one component
-    String[] components = filename.split(Path.SEPARATOR);
-    // get the first component
-    if (pathPattern.isAbsolute()) {
-      parents[0] = new Path(Path.SEPARATOR);
-      level = 1;
-    } else {
-      parents[0] = new Path(Path.CUR_DIR);
+    
+    // seed the parent directory path, return if it doesn't exist
+    try {
+      matches.add(getFileStatus(new Path(baseDir)));
+    } catch (FileNotFoundException e) {
+      return patternHasGlob ? matches : null;
     }
-
-    // glob the paths that match the parent path, i.e., [0, components.length-1]
-    boolean[] hasGlob = new boolean[]{false};
-    Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob);
-    FileStatus[] results;
-    if (parentPaths == null || parentPaths.length == 0) {
-      results = null;
-    } else {
-      // Now work on the last component of the path
-      GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
-      if (fp.hasPattern()) { // last component has a pattern
-        // list parent directories and then glob the results
-        try {
-          results = listStatus(parentPaths, fp);
-        } catch (FileNotFoundException e) {
-          results = null;
-        }
-        hasGlob[0] = true;
-      } else { // last component does not have a pattern
-        // remove the quoting of metachars in a non-regexp expansion
-        String name = unquotePathComponent(components[components.length - 1]);
-        // get all the path names
-        ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
-        for (int i = 0; i < parentPaths.length; i++) {
-          parentPaths[i] = new Path(parentPaths[i], name);
-          if (fp.accept(parentPaths[i])) {
-            filteredPaths.add(parentPaths[i]);
+    
+    // skip if there are no components other than the basedir
+    if (components != null) {
+      // iterate through each path component
+      for (int i=level; (i < components.length) && !matches.isEmpty(); i++) {
+        List<FileStatus> children = new ArrayList<FileStatus>();
+        for (FileStatus match : matches) {
+          // don't look for children in a file matched by a glob
+          if (!match.isDirectory()) {
+            continue;
+          }
+          try {
+            if (filters[i].hasPattern()) {
+              // get all children matching the filter
+              FileStatus[] statuses = listStatus(match.getPath(), filters[i]);
+              children.addAll(Arrays.asList(statuses));
+            } else {
+              // the component does not have a pattern
+              String component = unquotePathComponent(components[i]);
+              Path child = new Path(match.getPath(), component);
+              children.add(getFileStatus(child));
+            }
+          } catch (FileNotFoundException e) {
+            // don't care
           }
         }
-        // get all their statuses
-        results = getFileStatus(
-            filteredPaths.toArray(new Path[filteredPaths.size()]));
+        matches = children;
       }
     }
-
-    // Decide if the pathPattern contains a glob or not
-    if (results == null) {
-      if (hasGlob[0]) {
-        results = new FileStatus[0];
-      }
-    } else {
-      if (results.length == 0 ) {
-        if (!hasGlob[0]) {
-          results = null;
+    // remove anything that didn't match the filter
+    if (!matches.isEmpty()) {
+      Iterator<FileStatus> iter = matches.iterator();
+      while (iter.hasNext()) {
+        if (!filter.accept(iter.next().getPath())) {
+          iter.remove();
         }
-      } else {
-        Arrays.sort(results);
       }
     }
-    return results;
-  }
-
-  /*
-   * For a path of N components, return a list of paths that match the
-   * components [<code>level</code>, <code>N-1</code>].
-   */
-  private Path[] globPathsLevel(Path[] parents, String[] filePattern,
-      int level, boolean[] hasGlob) throws IOException {
-    if (level == filePattern.length - 1)
-      return parents;
-    if (parents == null || parents.length == 0) {
-      return null;
-    }
-    GlobFilter fp = new GlobFilter(filePattern[level]);
-    if (fp.hasPattern()) {
-      try {
-        parents = FileUtil.stat2Paths(listStatus(parents, fp));
-      } catch (FileNotFoundException e) {
-        parents = null;
-      }
-      hasGlob[0] = true;
-    } else { // the component does not have a pattern
-      // remove the quoting of metachars in a non-regexp expansion
-      String name = unquotePathComponent(filePattern[level]);
-      for (int i = 0; i < parents.length; i++) {
-        parents[i] = new Path(parents[i], name);
-      }
+    // no final paths, if there were any globs return empty list
+    if (matches.isEmpty()) {
+      return patternHasGlob ? matches : null;
     }
-    return globPathsLevel(parents, filePattern, level + 1, hasGlob);
+    Collections.sort(matches);
+    return matches;
   }
 
   /**
@@ -2164,30 +2157,6 @@ public abstract class FileSystem extends Configured implements Closeable {
     //doesn't do anything
   }
 
-  /**
-   * Return a list of file status objects that corresponds to the list of paths
-   * excluding those non-existent paths.
-   * 
-   * @param paths
-   *          the list of paths we want information from
-   * @return a list of FileStatus objects
-   * @throws IOException
-   *           see specific implementation
-   */
-  private FileStatus[] getFileStatus(Path[] paths) throws IOException {
-    if (paths == null) {
-      return null;
-    }
-    ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
-    for (int i = 0; i < paths.length; i++) {
-      try {
-        results.add(getFileStatus(paths[i]));
-      } catch (FileNotFoundException e) { // do nothing
-      }
-    }
-    return results.toArray(new FileStatus[results.size()]);
-  }
-  
   /**
    * Returns a status object describing the use and capacity of the
    * file system. If the file system has multiple partitions, the

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java

@@ -39,12 +39,26 @@ class GlobExpander {
   }
   
   /**
-   * Expand globs in the given <code>filePattern</code> into a collection of 
-   * file patterns so that in the expanded set no file pattern has a
-   * slash character ("/") in a curly bracket pair.
+   * Expand globs in the given <code>filePattern</code> into a collection of
+   * file patterns so that in the expanded set no file pattern has a slash
+   * character ("/") in a curly bracket pair.
+   * <p>
+   * Some examples of how the filePattern is expanded:<br>
+   * <pre>
+   * <b>
+   * filePattern         - Expanded file pattern </b>
+   * {a/b}               - a/b
+   * /}{a/b}             - /}a/b
+   * p{a/b,c/d}s         - pa/bs, pc/ds
+   * {a/b,c/d,{e,f}}     - a/b, c/d, {e,f}
+   * {a/b,c/d}{e,f}      - a/b{e,f}, c/d{e,f}
+   * {a,b}/{b,{c/d,e/f}} - {a,b}/b, {a,b}/c/d, {a,b}/e/f
+   * {a,b}/{c/\d}        - {a,b}/c/d
+   * </pre>
+   * 
    * @param filePattern
    * @return expanded file patterns
-   * @throws IOException 
+   * @throws IOException
    */
   public static List<String> expand(String filePattern) throws IOException {
     List<String> fullyExpanded = new ArrayList<String>();
@@ -65,7 +79,7 @@ class GlobExpander {
   /**
    * Expand the leftmost outer curly bracket pair containing a
    * slash character ("/") in <code>filePattern</code>.
-   * @param filePattern
+   * @param filePatternWithOffset
    * @return expanded file patterns
    * @throws IOException 
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java

@@ -387,7 +387,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
       copyBytesToLocal(n);       // modifies userBufLen, etc.
       if (localBufOff >= 4) {    // should be strictly ==
         long inputSize = readUIntLE(localBuf, 0);
-        if (inputSize != (inflater.getBytesWritten() & 0xffffffff)) {
+        if (inputSize != (inflater.getBytesWritten() & 0xffffffffL)) {
           throw new IOException(
             "stored gzip size doesn't match decompressed size");
         }
@@ -571,7 +571,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
     return ((((long)(b[off+3] & 0xff) << 24) |
              ((long)(b[off+2] & 0xff) << 16) |
              ((long)(b[off+1] & 0xff) <<  8) |
-             ((long)(b[off]   & 0xff)      )) & 0xffffffff);
+             ((long)(b[off]   & 0xff)      )) & 0xffffffffL);
   }
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -225,7 +225,6 @@ public class Client {
     private IpcConnectionContextProto connectionContext;   // connection context
     private final ConnectionId remoteId;                // connection id
     private AuthMethod authMethod; // authentication method
-    private boolean useSasl;
     private Token<? extends TokenIdentifier> token;
     private SaslRpcClient saslRpcClient;
     
@@ -270,8 +269,7 @@ public class Client {
 
       UserGroupInformation ticket = remoteId.getTicket();
       Class<?> protocol = remoteId.getProtocol();
-      this.useSasl = UserGroupInformation.isSecurityEnabled();
-      if (useSasl && protocol != null) {
+      if (protocol != null) {
         TokenInfo tokenInfo = SecurityUtil.getTokenInfo(protocol, conf);
         if (tokenInfo != null) {
           TokenSelector<? extends TokenIdentifier> tokenSelector = null;
@@ -296,12 +294,12 @@ public class Client {
         }
       }
       
-      if (!useSasl) {
-        authMethod = AuthMethod.SIMPLE;
-      } else if (token != null) {
+      if (token != null) {
         authMethod = AuthMethod.DIGEST;
-      } else {
+      } else if (UserGroupInformation.isSecurityEnabled()) {
         authMethod = AuthMethod.KERBEROS;
+      } else {
+        authMethod = AuthMethod.SIMPLE;
       }
       
       connectionContext = ProtoUtil.makeIpcConnectionContext(
@@ -576,14 +574,12 @@ public class Client {
           InputStream inStream = NetUtils.getInputStream(socket);
           OutputStream outStream = NetUtils.getOutputStream(socket);
           writeConnectionHeader(outStream);
-          if (useSasl) {
+          if (authMethod != AuthMethod.SIMPLE) {
             final InputStream in2 = inStream;
             final OutputStream out2 = outStream;
             UserGroupInformation ticket = remoteId.getTicket();
-            if (authMethod == AuthMethod.KERBEROS) {
-              if (ticket.getRealUser() != null) {
-                ticket = ticket.getRealUser();
-              }
+            if (ticket.getRealUser() != null) {
+              ticket = ticket.getRealUser();
             }
             boolean continueSasl = false;
             try {
@@ -614,7 +610,6 @@ public class Client {
                   connectionContext.getProtocol(), 
                   ProtoUtil.getUgi(connectionContext.getUserInfo()),
                   authMethod);
-              useSasl = false;
             }
           }
         
@@ -1174,7 +1169,7 @@ public class Client {
                   call.error);
         }
       } else {
-        return call.rpcResponse;
+        return call.getRpcResult();
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java

@@ -113,12 +113,17 @@ import org.codehaus.jackson.JsonGenerator;
  *  All other objects will be converted to a string and output as such.
  *  
  *  The bean's name and modelerType will be returned for all beans.
+ *
+ *  Optional paramater "callback" should be used to deliver JSONP response.
+ *  
  */
 public class JMXJsonServlet extends HttpServlet {
   private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
 
   private static final long serialVersionUID = 1L;
 
+  private static final String CALLBACK_PARAM = "callback";
+
   /**
    * MBean server.
    */
@@ -154,11 +159,22 @@ public class JMXJsonServlet extends HttpServlet {
         return;
       }
       JsonGenerator jg = null;
+      String jsonpcb = null;
+      PrintWriter writer = null;
       try {
-        response.setContentType("application/json; charset=utf8");
+        writer = response.getWriter();
+ 
+        // "callback" parameter implies JSONP outpout
+        jsonpcb = request.getParameter(CALLBACK_PARAM);
+        if (jsonpcb != null) {
+          response.setContentType("application/javascript; charset=utf8");
+          writer.write(jsonpcb + "(");
+        } else {
+          response.setContentType("application/json; charset=utf8");
+        }
 
-        PrintWriter writer = response.getWriter();
         jg = jsonFactory.createJsonGenerator(writer);
+        jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
         jg.useDefaultPrettyPrinter();
         jg.writeStartObject();
 
@@ -188,6 +204,12 @@ public class JMXJsonServlet extends HttpServlet {
         if (jg != null) {
           jg.close();
         }
+        if (jsonpcb != null) {
+           writer.write(");");
+        }
+        if (writer != null) {
+          writer.close();
+        }
       }
     } catch (IOException e) {
       LOG.error("Caught an exception while processing JMX request", e);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ContextFactory.java

@@ -1,211 +1,211 @@
-/*
- * ContextFactory.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.spi.NullContext;
-
-/**
- * Factory class for creating MetricsContext objects.  To obtain an instance
- * of this class, use the static <code>getFactory()</code> method.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class ContextFactory {
-    
-  private static final String PROPERTIES_FILE = 
-    "/hadoop-metrics.properties";
-  private static final String CONTEXT_CLASS_SUFFIX =
-    ".class";
-  private static final String DEFAULT_CONTEXT_CLASSNAME =
-    "org.apache.hadoop.metrics.spi.NullContext";
-    
-  private static ContextFactory theFactory = null;
-    
-  private Map<String,Object> attributeMap = new HashMap<String,Object>();
-  private Map<String,MetricsContext> contextMap = 
-    new HashMap<String,MetricsContext>();
-    
-  // Used only when contexts, or the ContextFactory itself, cannot be
-  // created.
-  private static Map<String,MetricsContext> nullContextMap = 
-    new HashMap<String,MetricsContext>();
-    
-  /** Creates a new instance of ContextFactory */
-  protected ContextFactory() {
-  }
-    
-  /**
-   * Returns the value of the named attribute, or null if there is no 
-   * attribute of that name.
-   *
-   * @param attributeName the attribute name
-   * @return the attribute value
-   */
-  public Object getAttribute(String attributeName) {
-    return attributeMap.get(attributeName);
-  }
-    
-  /**
-   * Returns the names of all the factory's attributes.
-   * 
-   * @return the attribute names
-   */
-  public String[] getAttributeNames() {
-    String[] result = new String[attributeMap.size()];
-    int i = 0;
-    // for (String attributeName : attributeMap.keySet()) {
-    Iterator it = attributeMap.keySet().iterator();
-    while (it.hasNext()) {
-      result[i++] = (String) it.next();
-    }
-    return result;
-  }
-    
-  /**
-   * Sets the named factory attribute to the specified value, creating it
-   * if it did not already exist.  If the value is null, this is the same as
-   * calling removeAttribute.
-   *
-   * @param attributeName the attribute name
-   * @param value the new attribute value
-   */
-  public void setAttribute(String attributeName, Object value) {
-    attributeMap.put(attributeName, value);
-  }
-
-  /**
-   * Removes the named attribute if it exists.
-   *
-   * @param attributeName the attribute name
-   */
-  public void removeAttribute(String attributeName) {
-    attributeMap.remove(attributeName);
-  }
-    
-  /**
-   * Returns the named MetricsContext instance, constructing it if necessary 
-   * using the factory's current configuration attributes. <p/>
-   * 
-   * When constructing the instance, if the factory property 
-   * <i>contextName</i>.class</code> exists, 
-   * its value is taken to be the name of the class to instantiate.  Otherwise,
-   * the default is to create an instance of 
-   * <code>org.apache.hadoop.metrics.spi.NullContext</code>, which is a 
-   * dummy "no-op" context which will cause all metric data to be discarded.
-   * 
-   * @param contextName the name of the context
-   * @return the named MetricsContext
-   */
-  public synchronized MetricsContext getContext(String refName, String contextName)
-      throws IOException, ClassNotFoundException,
-             InstantiationException, IllegalAccessException {
-    MetricsContext metricsContext = contextMap.get(refName);
-    if (metricsContext == null) {
-      String classNameAttribute = refName + CONTEXT_CLASS_SUFFIX;
-      String className = (String) getAttribute(classNameAttribute);
-      if (className == null) {
-        className = DEFAULT_CONTEXT_CLASSNAME;
-      }
-      Class contextClass = Class.forName(className);
-      metricsContext = (MetricsContext) contextClass.newInstance();
-      metricsContext.init(contextName, this);
-      contextMap.put(contextName, metricsContext);
-    }
-    return metricsContext;
-  }
-
-  public synchronized MetricsContext getContext(String contextName)
-    throws IOException, ClassNotFoundException, InstantiationException,
-           IllegalAccessException {
-    return getContext(contextName, contextName);
-  }
-  
-  /** 
-   * Returns all MetricsContexts built by this factory.
-   */
-  public synchronized Collection<MetricsContext> getAllContexts() {
-    // Make a copy to avoid race conditions with creating new contexts.
-    return new ArrayList<MetricsContext>(contextMap.values());
-  }
-    
-  /**
-   * Returns a "null" context - one which does nothing.
-   */
-  public static synchronized MetricsContext getNullContext(String contextName) {
-    MetricsContext nullContext = nullContextMap.get(contextName);
-    if (nullContext == null) {
-      nullContext = new NullContext();
-      nullContextMap.put(contextName, nullContext);
-    }
-    return nullContext;
-  }
-    
-  /**
-   * Returns the singleton ContextFactory instance, constructing it if 
-   * necessary. <p/>
-   * 
-   * When the instance is constructed, this method checks if the file 
-   * <code>hadoop-metrics.properties</code> exists on the class path.  If it 
-   * exists, it must be in the format defined by java.util.Properties, and all 
-   * the properties in the file are set as attributes on the newly created
-   * ContextFactory instance.
-   *
-   * @return the singleton ContextFactory instance
-   */
-  public static synchronized ContextFactory getFactory() throws IOException {
-    if (theFactory == null) {
-      theFactory = new ContextFactory();
-      theFactory.setAttributes();
-    }
-    return theFactory;
-  }
-    
-  private void setAttributes() throws IOException {
-    InputStream is = getClass().getResourceAsStream(PROPERTIES_FILE);
-    if (is != null) {
-      try {
-        Properties properties = new Properties();
-        properties.load(is);
-        //for (Object propertyNameObj : properties.keySet()) {
-        Iterator it = properties.keySet().iterator();
-        while (it.hasNext()) {
-          String propertyName = (String) it.next();
-          String propertyValue = properties.getProperty(propertyName);
-          setAttribute(propertyName, propertyValue);
-        }
-      } finally {
-        is.close();
-      }
-    }
-  }
-    
-}
+/*
+ * ContextFactory.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics.spi.NullContext;
+
+/**
+ * Factory class for creating MetricsContext objects.  To obtain an instance
+ * of this class, use the static <code>getFactory()</code> method.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class ContextFactory {
+    
+  private static final String PROPERTIES_FILE = 
+    "/hadoop-metrics.properties";
+  private static final String CONTEXT_CLASS_SUFFIX =
+    ".class";
+  private static final String DEFAULT_CONTEXT_CLASSNAME =
+    "org.apache.hadoop.metrics.spi.NullContext";
+    
+  private static ContextFactory theFactory = null;
+    
+  private Map<String,Object> attributeMap = new HashMap<String,Object>();
+  private Map<String,MetricsContext> contextMap = 
+    new HashMap<String,MetricsContext>();
+    
+  // Used only when contexts, or the ContextFactory itself, cannot be
+  // created.
+  private static Map<String,MetricsContext> nullContextMap = 
+    new HashMap<String,MetricsContext>();
+    
+  /** Creates a new instance of ContextFactory */
+  protected ContextFactory() {
+  }
+    
+  /**
+   * Returns the value of the named attribute, or null if there is no 
+   * attribute of that name.
+   *
+   * @param attributeName the attribute name
+   * @return the attribute value
+   */
+  public Object getAttribute(String attributeName) {
+    return attributeMap.get(attributeName);
+  }
+    
+  /**
+   * Returns the names of all the factory's attributes.
+   * 
+   * @return the attribute names
+   */
+  public String[] getAttributeNames() {
+    String[] result = new String[attributeMap.size()];
+    int i = 0;
+    // for (String attributeName : attributeMap.keySet()) {
+    Iterator it = attributeMap.keySet().iterator();
+    while (it.hasNext()) {
+      result[i++] = (String) it.next();
+    }
+    return result;
+  }
+    
+  /**
+   * Sets the named factory attribute to the specified value, creating it
+   * if it did not already exist.  If the value is null, this is the same as
+   * calling removeAttribute.
+   *
+   * @param attributeName the attribute name
+   * @param value the new attribute value
+   */
+  public void setAttribute(String attributeName, Object value) {
+    attributeMap.put(attributeName, value);
+  }
+
+  /**
+   * Removes the named attribute if it exists.
+   *
+   * @param attributeName the attribute name
+   */
+  public void removeAttribute(String attributeName) {
+    attributeMap.remove(attributeName);
+  }
+    
+  /**
+   * Returns the named MetricsContext instance, constructing it if necessary 
+   * using the factory's current configuration attributes. <p/>
+   * 
+   * When constructing the instance, if the factory property 
+   * <i>contextName</i>.class</code> exists, 
+   * its value is taken to be the name of the class to instantiate.  Otherwise,
+   * the default is to create an instance of 
+   * <code>org.apache.hadoop.metrics.spi.NullContext</code>, which is a 
+   * dummy "no-op" context which will cause all metric data to be discarded.
+   * 
+   * @param contextName the name of the context
+   * @return the named MetricsContext
+   */
+  public synchronized MetricsContext getContext(String refName, String contextName)
+      throws IOException, ClassNotFoundException,
+             InstantiationException, IllegalAccessException {
+    MetricsContext metricsContext = contextMap.get(refName);
+    if (metricsContext == null) {
+      String classNameAttribute = refName + CONTEXT_CLASS_SUFFIX;
+      String className = (String) getAttribute(classNameAttribute);
+      if (className == null) {
+        className = DEFAULT_CONTEXT_CLASSNAME;
+      }
+      Class contextClass = Class.forName(className);
+      metricsContext = (MetricsContext) contextClass.newInstance();
+      metricsContext.init(contextName, this);
+      contextMap.put(contextName, metricsContext);
+    }
+    return metricsContext;
+  }
+
+  public synchronized MetricsContext getContext(String contextName)
+    throws IOException, ClassNotFoundException, InstantiationException,
+           IllegalAccessException {
+    return getContext(contextName, contextName);
+  }
+  
+  /** 
+   * Returns all MetricsContexts built by this factory.
+   */
+  public synchronized Collection<MetricsContext> getAllContexts() {
+    // Make a copy to avoid race conditions with creating new contexts.
+    return new ArrayList<MetricsContext>(contextMap.values());
+  }
+    
+  /**
+   * Returns a "null" context - one which does nothing.
+   */
+  public static synchronized MetricsContext getNullContext(String contextName) {
+    MetricsContext nullContext = nullContextMap.get(contextName);
+    if (nullContext == null) {
+      nullContext = new NullContext();
+      nullContextMap.put(contextName, nullContext);
+    }
+    return nullContext;
+  }
+    
+  /**
+   * Returns the singleton ContextFactory instance, constructing it if 
+   * necessary. <p/>
+   * 
+   * When the instance is constructed, this method checks if the file 
+   * <code>hadoop-metrics.properties</code> exists on the class path.  If it 
+   * exists, it must be in the format defined by java.util.Properties, and all 
+   * the properties in the file are set as attributes on the newly created
+   * ContextFactory instance.
+   *
+   * @return the singleton ContextFactory instance
+   */
+  public static synchronized ContextFactory getFactory() throws IOException {
+    if (theFactory == null) {
+      theFactory = new ContextFactory();
+      theFactory.setAttributes();
+    }
+    return theFactory;
+  }
+    
+  private void setAttributes() throws IOException {
+    InputStream is = getClass().getResourceAsStream(PROPERTIES_FILE);
+    if (is != null) {
+      try {
+        Properties properties = new Properties();
+        properties.load(is);
+        //for (Object propertyNameObj : properties.keySet()) {
+        Iterator it = properties.keySet().iterator();
+        while (it.hasNext()) {
+          String propertyName = (String) it.next();
+          String propertyValue = properties.getProperty(propertyName);
+          setAttribute(propertyName, propertyValue);
+        }
+      } finally {
+        is.close();
+      }
+    }
+  }
+    
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsContext.java

@@ -1,122 +1,122 @@
-/*
- * MetricsContext.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-
-/**
- * The main interface to the metrics package. 
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface MetricsContext {
-    
-  /**
-   * Default period in seconds at which data is sent to the metrics system.
-   */
-  public static final int DEFAULT_PERIOD = 5;
-
-  /**
-   * Initialize this context.
-   * @param contextName The given name for this context
-   * @param factory The creator of this context
-   */
-  public void init(String contextName, ContextFactory factory);
-
-  /**
-   * Returns the context name.
-   *
-   * @return the context name
-   */
-  public abstract String getContextName();
-    
-  /**
-   * Starts or restarts monitoring, the emitting of metrics records as they are 
-   * updated. 
-   */
-  public abstract void startMonitoring()
-    throws IOException;
-
-  /**
-   * Stops monitoring.  This does not free any data that the implementation
-   * may have buffered for sending at the next timer event. It
-   * is OK to call <code>startMonitoring()</code> again after calling 
-   * this.
-   * @see #close()
-   */
-  public abstract void stopMonitoring();
-    
-  /**
-   * Returns true if monitoring is currently in progress.
-   */
-  public abstract boolean isMonitoring();
-    
-  /**
-   * Stops monitoring and also frees any buffered data, returning this 
-   * object to its initial state.  
-   */
-  public abstract void close();
-    
-  /**
-   * Creates a new MetricsRecord instance with the given <code>recordName</code>.
-   * Throws an exception if the metrics implementation is configured with a fixed
-   * set of record names and <code>recordName</code> is not in that set.
-   *
-   * @param recordName the name of the record
-   * @throws MetricsException if recordName conflicts with configuration data
-   */
-  public abstract MetricsRecord createRecord(String recordName);
-    
-  /**
-   * Registers a callback to be called at regular time intervals, as 
-   * determined by the implementation-class specific configuration.
-   *
-   * @param updater object to be run periodically; it should updated
-   * some metrics records and then return
-   */
-  public abstract void registerUpdater(Updater updater);
-
-  /**
-   * Removes a callback, if it exists.
-   * 
-   * @param updater object to be removed from the callback list
-   */
-  public abstract void unregisterUpdater(Updater updater);
-  
-  /**
-   * Returns the timer period.
-   */
-  public abstract int getPeriod();
-  
-  /**
-   * Retrieves all the records managed by this MetricsContext.
-   * Useful for monitoring systems that are polling-based.
-   * 
-   * @return A non-null map from all record names to the records managed.
-   */
-   Map<String, Collection<OutputRecord>> getAllRecords();
-}
+/*
+ * MetricsContext.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics.spi.OutputRecord;
+
+/**
+ * The main interface to the metrics package. 
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface MetricsContext {
+    
+  /**
+   * Default period in seconds at which data is sent to the metrics system.
+   */
+  public static final int DEFAULT_PERIOD = 5;
+
+  /**
+   * Initialize this context.
+   * @param contextName The given name for this context
+   * @param factory The creator of this context
+   */
+  public void init(String contextName, ContextFactory factory);
+
+  /**
+   * Returns the context name.
+   *
+   * @return the context name
+   */
+  public abstract String getContextName();
+    
+  /**
+   * Starts or restarts monitoring, the emitting of metrics records as they are 
+   * updated. 
+   */
+  public abstract void startMonitoring()
+    throws IOException;
+
+  /**
+   * Stops monitoring.  This does not free any data that the implementation
+   * may have buffered for sending at the next timer event. It
+   * is OK to call <code>startMonitoring()</code> again after calling 
+   * this.
+   * @see #close()
+   */
+  public abstract void stopMonitoring();
+    
+  /**
+   * Returns true if monitoring is currently in progress.
+   */
+  public abstract boolean isMonitoring();
+    
+  /**
+   * Stops monitoring and also frees any buffered data, returning this 
+   * object to its initial state.  
+   */
+  public abstract void close();
+    
+  /**
+   * Creates a new MetricsRecord instance with the given <code>recordName</code>.
+   * Throws an exception if the metrics implementation is configured with a fixed
+   * set of record names and <code>recordName</code> is not in that set.
+   *
+   * @param recordName the name of the record
+   * @throws MetricsException if recordName conflicts with configuration data
+   */
+  public abstract MetricsRecord createRecord(String recordName);
+    
+  /**
+   * Registers a callback to be called at regular time intervals, as 
+   * determined by the implementation-class specific configuration.
+   *
+   * @param updater object to be run periodically; it should updated
+   * some metrics records and then return
+   */
+  public abstract void registerUpdater(Updater updater);
+
+  /**
+   * Removes a callback, if it exists.
+   * 
+   * @param updater object to be removed from the callback list
+   */
+  public abstract void unregisterUpdater(Updater updater);
+  
+  /**
+   * Returns the timer period.
+   */
+  public abstract int getPeriod();
+  
+  /**
+   * Retrieves all the records managed by this MetricsContext.
+   * Useful for monitoring systems that are polling-based.
+   * 
+   * @return A non-null map from all record names to the records managed.
+   */
+   Map<String, Collection<OutputRecord>> getAllRecords();
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsException.java

@@ -1,47 +1,47 @@
-/*
- * MetricsException.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * General-purpose, unchecked metrics exception.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class MetricsException extends RuntimeException {
-    
-  private static final long serialVersionUID = -1643257498540498497L;
-
-  /** Creates a new instance of MetricsException */
-  public MetricsException() {
-  }
-    
-  /** Creates a new instance of MetricsException 
-   *
-   * @param message an error message
-   */
-  public MetricsException(String message) {
-    super(message);
-  }
-    
-}
+/*
+ * MetricsException.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * General-purpose, unchecked metrics exception.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class MetricsException extends RuntimeException {
+    
+  private static final long serialVersionUID = -1643257498540498497L;
+
+  /** Creates a new instance of MetricsException */
+  public MetricsException() {
+  }
+    
+  /** Creates a new instance of MetricsException 
+   *
+   * @param message an error message
+   */
+  public MetricsException(String message) {
+    super(message);
+  }
+    
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsRecord.java

@@ -1,251 +1,251 @@
-/*
- * MetricsRecord.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A named and optionally tagged set of records to be sent to the metrics
- * system. <p/>
- *
- * A record name identifies the kind of data to be reported. For example, a
- * program reporting statistics relating to the disks on a computer might use
- * a record name "diskStats".<p/>
- *
- * A record has zero or more <i>tags</i>. A tag has a name and a value. To
- * continue the example, the "diskStats" record might use a tag named
- * "diskName" to identify a particular disk.  Sometimes it is useful to have
- * more than one tag, so there might also be a "diskType" with value "ide" or
- * "scsi" or whatever.<p/>
- *
- * A record also has zero or more <i>metrics</i>.  These are the named
- * values that are to be reported to the metrics system.  In the "diskStats"
- * example, possible metric names would be "diskPercentFull", "diskPercentBusy", 
- * "kbReadPerSecond", etc.<p/>
- * 
- * The general procedure for using a MetricsRecord is to fill in its tag and
- * metric values, and then call <code>update()</code> to pass the record to the
- * client library.
- * Metric data is not immediately sent to the metrics system
- * each time that <code>update()</code> is called. 
- * An internal table is maintained, identified by the record name. This
- * table has columns 
- * corresponding to the tag and the metric names, and rows 
- * corresponding to each unique set of tag values. An update
- * either modifies an existing row in the table, or adds a new row with a set of
- * tag values that are different from all the other rows.  Note that if there
- * are no tags, then there can be at most one row in the table. <p/>
- * 
- * Once a row is added to the table, its data will be sent to the metrics system 
- * on every timer period, whether or not it has been updated since the previous
- * timer period.  If this is inappropriate, for example if metrics were being
- * reported by some transient object in an application, the <code>remove()</code>
- * method can be used to remove the row and thus stop the data from being
- * sent.<p/>
- *
- * Note that the <code>update()</code> method is atomic.  This means that it is
- * safe for different threads to be updating the same metric.  More precisely,
- * it is OK for different threads to call <code>update()</code> on MetricsRecord instances 
- * with the same set of tag names and tag values.  Different threads should 
- * <b>not</b> use the same MetricsRecord instance at the same time.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface MetricsRecord {
-    
-  /**
-   * Returns the record name. 
-   *
-   * @return the record name
-   */
-  public abstract String getRecordName();
-    
-  /**
-   * Sets the named tag to the specified value.  The tagValue may be null, 
-   * which is treated the same as an empty String.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  public abstract void setTag(String tagName, String tagValue);
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  public abstract void setTag(String tagName, int tagValue);
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  public abstract void setTag(String tagName, long tagValue);
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  public abstract void setTag(String tagName, short tagValue);
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  public abstract void setTag(String tagName, byte tagValue);
-    
-  /**
-   * Removes any tag of the specified name.
-   *
-   * @param tagName name of a tag
-   */
-  public abstract void removeTag(String tagName);
-  
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void setMetric(String metricName, int metricValue);
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void setMetric(String metricName, long metricValue);
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void setMetric(String metricName, short metricValue);
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void setMetric(String metricName, byte metricValue);
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void setMetric(String metricName, float metricValue);
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void incrMetric(String metricName, int metricValue);
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void incrMetric(String metricName, long metricValue);
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void incrMetric(String metricName, short metricValue);
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void incrMetric(String metricName, byte metricValue);
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  public abstract void incrMetric(String metricName, float metricValue);
-    
-  /**
-   * Updates the table of buffered data which is to be sent periodically.
-   * If the tag values match an existing row, that row is updated; 
-   * otherwise, a new row is added.
-   */
-  public abstract void update();
-    
-  /**
-   * Removes, from the buffered data table, all rows having tags 
-   * that equal the tags that have been set on this record. For example,
-   * if there are no tags on this record, all rows for this record name
-   * would be removed.  Or, if there is a single tag on this record, then
-   * just rows containing a tag with the same name and value would be removed.
-   */
-  public abstract void remove();
-    
-}
+/*
+ * MetricsRecord.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A named and optionally tagged set of records to be sent to the metrics
+ * system. <p/>
+ *
+ * A record name identifies the kind of data to be reported. For example, a
+ * program reporting statistics relating to the disks on a computer might use
+ * a record name "diskStats".<p/>
+ *
+ * A record has zero or more <i>tags</i>. A tag has a name and a value. To
+ * continue the example, the "diskStats" record might use a tag named
+ * "diskName" to identify a particular disk.  Sometimes it is useful to have
+ * more than one tag, so there might also be a "diskType" with value "ide" or
+ * "scsi" or whatever.<p/>
+ *
+ * A record also has zero or more <i>metrics</i>.  These are the named
+ * values that are to be reported to the metrics system.  In the "diskStats"
+ * example, possible metric names would be "diskPercentFull", "diskPercentBusy", 
+ * "kbReadPerSecond", etc.<p/>
+ * 
+ * The general procedure for using a MetricsRecord is to fill in its tag and
+ * metric values, and then call <code>update()</code> to pass the record to the
+ * client library.
+ * Metric data is not immediately sent to the metrics system
+ * each time that <code>update()</code> is called. 
+ * An internal table is maintained, identified by the record name. This
+ * table has columns 
+ * corresponding to the tag and the metric names, and rows 
+ * corresponding to each unique set of tag values. An update
+ * either modifies an existing row in the table, or adds a new row with a set of
+ * tag values that are different from all the other rows.  Note that if there
+ * are no tags, then there can be at most one row in the table. <p/>
+ * 
+ * Once a row is added to the table, its data will be sent to the metrics system 
+ * on every timer period, whether or not it has been updated since the previous
+ * timer period.  If this is inappropriate, for example if metrics were being
+ * reported by some transient object in an application, the <code>remove()</code>
+ * method can be used to remove the row and thus stop the data from being
+ * sent.<p/>
+ *
+ * Note that the <code>update()</code> method is atomic.  This means that it is
+ * safe for different threads to be updating the same metric.  More precisely,
+ * it is OK for different threads to call <code>update()</code> on MetricsRecord instances 
+ * with the same set of tag names and tag values.  Different threads should 
+ * <b>not</b> use the same MetricsRecord instance at the same time.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface MetricsRecord {
+    
+  /**
+   * Returns the record name. 
+   *
+   * @return the record name
+   */
+  public abstract String getRecordName();
+    
+  /**
+   * Sets the named tag to the specified value.  The tagValue may be null, 
+   * which is treated the same as an empty String.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  public abstract void setTag(String tagName, String tagValue);
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  public abstract void setTag(String tagName, int tagValue);
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  public abstract void setTag(String tagName, long tagValue);
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  public abstract void setTag(String tagName, short tagValue);
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  public abstract void setTag(String tagName, byte tagValue);
+    
+  /**
+   * Removes any tag of the specified name.
+   *
+   * @param tagName name of a tag
+   */
+  public abstract void removeTag(String tagName);
+  
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void setMetric(String metricName, int metricValue);
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void setMetric(String metricName, long metricValue);
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void setMetric(String metricName, short metricValue);
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void setMetric(String metricName, byte metricValue);
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void setMetric(String metricName, float metricValue);
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void incrMetric(String metricName, int metricValue);
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void incrMetric(String metricName, long metricValue);
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void incrMetric(String metricName, short metricValue);
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void incrMetric(String metricName, byte metricValue);
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  public abstract void incrMetric(String metricName, float metricValue);
+    
+  /**
+   * Updates the table of buffered data which is to be sent periodically.
+   * If the tag values match an existing row, that row is updated; 
+   * otherwise, a new row is added.
+   */
+  public abstract void update();
+    
+  /**
+   * Removes, from the buffered data table, all rows having tags 
+   * that equal the tags that have been set on this record. For example,
+   * if there are no tags on this record, all rows for this record name
+   * would be removed.  Or, if there is a single tag on this record, then
+   * just rows containing a tag with the same name and value would be removed.
+   */
+  public abstract void remove();
+    
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/file/FileContext.java

@@ -1,159 +1,159 @@
-/*
- * FileContext.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics.file;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-
-/**
- * Metrics context for writing metrics to a file.<p/>
- *
- * This class is configured by setting ContextFactory attributes which in turn
- * are usually configured through a properties file.  All the attributes are
- * prefixed by the contextName. For example, the properties file might contain:
- * <pre>
- * myContextName.fileName=/tmp/metrics.log
- * myContextName.period=5
- * </pre>
- * @see org.apache.hadoop.metrics2.sink.FileSink for metrics 2.0.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-@Deprecated
-public class FileContext extends AbstractMetricsContext {
-    
-  /* Configuration attribute names */
-  @InterfaceAudience.Private
-  protected static final String FILE_NAME_PROPERTY = "fileName";
-  @InterfaceAudience.Private
-  protected static final String PERIOD_PROPERTY = "period";
-    
-  private File file = null;              // file for metrics to be written to
-  private PrintWriter writer = null;
-    
-  /** Creates a new instance of FileContext */
-  @InterfaceAudience.Private
-  public FileContext() {}
-    
-  @Override
-  @InterfaceAudience.Private
-  public void init(String contextName, ContextFactory factory) {
-    super.init(contextName, factory);
-        
-    String fileName = getAttribute(FILE_NAME_PROPERTY);
-    if (fileName != null) {
-      file = new File(fileName);
-    }
-        
-    parseAndSetPeriod(PERIOD_PROPERTY);
-  }
-
-  /**
-   * Returns the configured file name, or null.
-   */
-  @InterfaceAudience.Private
-  public String getFileName() {
-    if (file == null) {
-      return null;
-    } else {
-      return file.getName();
-    }
-  }
-    
-  /**
-   * Starts or restarts monitoring, by opening in append-mode, the
-   * file specified by the <code>fileName</code> attribute,
-   * if specified. Otherwise the data will be written to standard
-   * output.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void startMonitoring()
-    throws IOException 
-  {
-    if (file == null) {
-      writer = new PrintWriter(new BufferedOutputStream(System.out));
-    } else {
-      writer = new PrintWriter(new FileWriter(file, true));
-    }
-    super.startMonitoring();
-  }
-    
-  /**
-   * Stops monitoring, closing the file.
-   * @see #close()
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void stopMonitoring() {
-    super.stopMonitoring();
-        
-    if (writer != null) {
-      writer.close();
-      writer = null;
-    }
-  }
-    
-  /**
-   * Emits a metrics record to a file.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void emitRecord(String contextName, String recordName, OutputRecord outRec) {
-    writer.print(contextName);
-    writer.print(".");
-    writer.print(recordName);
-    String separator = ": ";
-    for (String tagName : outRec.getTagNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(tagName);
-      writer.print("=");
-      writer.print(outRec.getTag(tagName));
-    }
-    for (String metricName : outRec.getMetricNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(metricName);
-      writer.print("=");
-      writer.print(outRec.getMetric(metricName));
-    }
-    writer.println();
-  }
-    
-  /**
-   * Flushes the output writer, forcing updates to disk.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void flush() {
-    writer.flush();
-  }
-}
+/*
+ * FileContext.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics.file;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics.ContextFactory;
+import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
+import org.apache.hadoop.metrics.spi.OutputRecord;
+
+/**
+ * Metrics context for writing metrics to a file.<p/>
+ *
+ * This class is configured by setting ContextFactory attributes which in turn
+ * are usually configured through a properties file.  All the attributes are
+ * prefixed by the contextName. For example, the properties file might contain:
+ * <pre>
+ * myContextName.fileName=/tmp/metrics.log
+ * myContextName.period=5
+ * </pre>
+ * @see org.apache.hadoop.metrics2.sink.FileSink for metrics 2.0.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@Deprecated
+public class FileContext extends AbstractMetricsContext {
+    
+  /* Configuration attribute names */
+  @InterfaceAudience.Private
+  protected static final String FILE_NAME_PROPERTY = "fileName";
+  @InterfaceAudience.Private
+  protected static final String PERIOD_PROPERTY = "period";
+    
+  private File file = null;              // file for metrics to be written to
+  private PrintWriter writer = null;
+    
+  /** Creates a new instance of FileContext */
+  @InterfaceAudience.Private
+  public FileContext() {}
+    
+  @Override
+  @InterfaceAudience.Private
+  public void init(String contextName, ContextFactory factory) {
+    super.init(contextName, factory);
+        
+    String fileName = getAttribute(FILE_NAME_PROPERTY);
+    if (fileName != null) {
+      file = new File(fileName);
+    }
+        
+    parseAndSetPeriod(PERIOD_PROPERTY);
+  }
+
+  /**
+   * Returns the configured file name, or null.
+   */
+  @InterfaceAudience.Private
+  public String getFileName() {
+    if (file == null) {
+      return null;
+    } else {
+      return file.getName();
+    }
+  }
+    
+  /**
+   * Starts or restarts monitoring, by opening in append-mode, the
+   * file specified by the <code>fileName</code> attribute,
+   * if specified. Otherwise the data will be written to standard
+   * output.
+   */
+  @Override
+  @InterfaceAudience.Private
+  public void startMonitoring()
+    throws IOException 
+  {
+    if (file == null) {
+      writer = new PrintWriter(new BufferedOutputStream(System.out));
+    } else {
+      writer = new PrintWriter(new FileWriter(file, true));
+    }
+    super.startMonitoring();
+  }
+    
+  /**
+   * Stops monitoring, closing the file.
+   * @see #close()
+   */
+  @Override
+  @InterfaceAudience.Private
+  public void stopMonitoring() {
+    super.stopMonitoring();
+        
+    if (writer != null) {
+      writer.close();
+      writer = null;
+    }
+  }
+    
+  /**
+   * Emits a metrics record to a file.
+   */
+  @Override
+  @InterfaceAudience.Private
+  public void emitRecord(String contextName, String recordName, OutputRecord outRec) {
+    writer.print(contextName);
+    writer.print(".");
+    writer.print(recordName);
+    String separator = ": ";
+    for (String tagName : outRec.getTagNames()) {
+      writer.print(separator);
+      separator = ", ";
+      writer.print(tagName);
+      writer.print("=");
+      writer.print(outRec.getTag(tagName));
+    }
+    for (String metricName : outRec.getMetricNames()) {
+      writer.print(separator);
+      separator = ", ";
+      writer.print(metricName);
+      writer.print("=");
+      writer.print(outRec.getMetric(metricName));
+    }
+    writer.println();
+  }
+    
+  /**
+   * Flushes the output writer, forcing updates to disk.
+   */
+  @Override
+  @InterfaceAudience.Private
+  public void flush() {
+    writer.flush();
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java

@@ -1,491 +1,491 @@
-/*
- * AbstractMetricsContext.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics.spi;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.apache.hadoop.metrics.MetricsException;
-import org.apache.hadoop.metrics.MetricsRecord;
-import org.apache.hadoop.metrics.Updater;
-
-/**
- * The main class of the Service Provider Interface.  This class should be
- * extended in order to integrate the Metrics API with a specific metrics
- * client library. <p/>
- *
- * This class implements the internal table of metric data, and the timer
- * on which data is to be sent to the metrics system.  Subclasses must
- * override the abstract <code>emitRecord</code> method in order to transmit
- * the data. <p/>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public abstract class AbstractMetricsContext implements MetricsContext {
-    
-  private int period = MetricsContext.DEFAULT_PERIOD;
-  private Timer timer = null;
-    
-  private Set<Updater> updaters = new HashSet<Updater>(1);
-  private volatile boolean isMonitoring = false;
-    
-  private ContextFactory factory = null;
-  private String contextName = null;
-    
-  @InterfaceAudience.Private
-  public static class TagMap extends TreeMap<String,Object> {
-    private static final long serialVersionUID = 3546309335061952993L;
-    TagMap() {
-      super();
-    }
-    TagMap(TagMap orig) {
-      super(orig);
-    }
-    /**
-     * Returns true if this tagmap contains every tag in other.
-     */
-    public boolean containsAll(TagMap other) {
-      for (Map.Entry<String,Object> entry : other.entrySet()) {
-        Object value = get(entry.getKey());
-        if (value == null || !value.equals(entry.getValue())) {
-          // either key does not exist here, or the value is different
-          return false;
-        }
-      }
-      return true;
-    }
-  }
-  
-  @InterfaceAudience.Private
-  public static class MetricMap extends TreeMap<String,Number> {
-    private static final long serialVersionUID = -7495051861141631609L;
-    MetricMap() {
-      super();
-    }
-    MetricMap(MetricMap orig) {
-      super(orig);
-    }
-  }
-            
-  static class RecordMap extends HashMap<TagMap,MetricMap> {
-    private static final long serialVersionUID = 259835619700264611L;
-  }
-    
-  private Map<String,RecordMap> bufferedData = new HashMap<String,RecordMap>();
-    
-
-  /**
-   * Creates a new instance of AbstractMetricsContext
-   */
-  protected AbstractMetricsContext() {
-  }
-    
-  /**
-   * Initializes the context.
-   */
-  @Override
-  public void init(String contextName, ContextFactory factory) 
-  {
-    this.contextName = contextName;
-    this.factory = factory;
-  }
-    
-  /**
-   * Convenience method for subclasses to access factory attributes.
-   */
-  protected String getAttribute(String attributeName) {
-    String factoryAttribute = contextName + "." + attributeName;
-    return (String) factory.getAttribute(factoryAttribute);  
-  }
-    
-  /**
-   * Returns an attribute-value map derived from the factory attributes
-   * by finding all factory attributes that begin with 
-   * <i>contextName</i>.<i>tableName</i>.  The returned map consists of
-   * those attributes with the contextName and tableName stripped off.
-   */
-  protected Map<String,String> getAttributeTable(String tableName) {
-    String prefix = contextName + "." + tableName + ".";
-    Map<String,String> result = new HashMap<String,String>();
-    for (String attributeName : factory.getAttributeNames()) {
-      if (attributeName.startsWith(prefix)) {
-        String name = attributeName.substring(prefix.length());
-        String value = (String) factory.getAttribute(attributeName);
-        result.put(name, value);
-      }
-    }
-    return result;
-  }
-    
-  /**
-   * Returns the context name.
-   */
-  @Override
-  public String getContextName() {
-    return contextName;
-  }
-    
-  /**
-   * Returns the factory by which this context was created.
-   */
-  public ContextFactory getContextFactory() {
-    return factory;
-  }
-    
-  /**
-   * Starts or restarts monitoring, the emitting of metrics records.
-   */
-  @Override
-  public synchronized void startMonitoring()
-    throws IOException {
-    if (!isMonitoring) {
-      startTimer();
-      isMonitoring = true;
-    }
-  }
-    
-  /**
-   * Stops monitoring.  This does not free buffered data. 
-   * @see #close()
-   */
-  @Override
-  public synchronized void stopMonitoring() {
-    if (isMonitoring) {
-      stopTimer();
-      isMonitoring = false;
-    }
-  }
-    
-  /**
-   * Returns true if monitoring is currently in progress.
-   */
-  @Override
-  public boolean isMonitoring() {
-    return isMonitoring;
-  }
-    
-  /**
-   * Stops monitoring and frees buffered data, returning this
-   * object to its initial state.  
-   */
-  @Override
-  public synchronized void close() {
-    stopMonitoring();
-    clearUpdaters();
-  } 
-    
-  /**
-   * Creates a new AbstractMetricsRecord instance with the given <code>recordName</code>.
-   * Throws an exception if the metrics implementation is configured with a fixed
-   * set of record names and <code>recordName</code> is not in that set.
-   * 
-   * @param recordName the name of the record
-   * @throws MetricsException if recordName conflicts with configuration data
-   */
-  @Override
-  public final synchronized MetricsRecord createRecord(String recordName) {
-    if (bufferedData.get(recordName) == null) {
-      bufferedData.put(recordName, new RecordMap());
-    }
-    return newRecord(recordName);
-  }
-    
-  /**
-   * Subclasses should override this if they subclass MetricsRecordImpl.
-   * @param recordName the name of the record
-   * @return newly created instance of MetricsRecordImpl or subclass
-   */
-  protected MetricsRecord newRecord(String recordName) {
-    return new MetricsRecordImpl(recordName, this);
-  }
-    
-  /**
-   * Registers a callback to be called at time intervals determined by
-   * the configuration.
-   *
-   * @param updater object to be run periodically; it should update
-   * some metrics records 
-   */
-  @Override
-  public synchronized void registerUpdater(final Updater updater) {
-    if (!updaters.contains(updater)) {
-      updaters.add(updater);
-    }
-  }
-    
-  /**
-   * Removes a callback, if it exists.
-   *
-   * @param updater object to be removed from the callback list
-   */
-  @Override
-  public synchronized void unregisterUpdater(Updater updater) {
-    updaters.remove(updater);
-  }
-    
-  private synchronized void clearUpdaters() {
-    updaters.clear();
-  }
-    
-  /**
-   * Starts timer if it is not already started
-   */
-  private synchronized void startTimer() {
-    if (timer == null) {
-      timer = new Timer("Timer thread for monitoring " + getContextName(), 
-                        true);
-      TimerTask task = new TimerTask() {
-          @Override
-          public void run() {
-            try {
-              timerEvent();
-            } catch (IOException ioe) {
-              ioe.printStackTrace();
-            }
-          }
-        };
-      long millis = period * 1000;
-      timer.scheduleAtFixedRate(task, millis, millis);
-    }
-  }
-    
-  /**
-   * Stops timer if it is running
-   */
-  private synchronized void stopTimer() {
-    if (timer != null) {
-      timer.cancel();
-      timer = null;
-    }
-  }
-    
-  /**
-   * Timer callback.
-   */
-  private void timerEvent() throws IOException {
-    if (isMonitoring) {
-      Collection<Updater> myUpdaters;
-      synchronized (this) {
-        myUpdaters = new ArrayList<Updater>(updaters);
-      }     
-      // Run all the registered updates without holding a lock
-      // on this context
-      for (Updater updater : myUpdaters) {
-        try {
-          updater.doUpdates(this);
-        } catch (Throwable throwable) {
-          throwable.printStackTrace();
-        }
-      }
-      emitRecords();
-    }
-  }
-    
-  /**
-   *  Emits the records.
-   */
-  private synchronized void emitRecords() throws IOException {
-    for (String recordName : bufferedData.keySet()) {
-      RecordMap recordMap = bufferedData.get(recordName);
-      synchronized (recordMap) {
-        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet ();
-        for (Entry<TagMap, MetricMap> entry : entrySet) {
-          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
-          emitRecord(contextName, recordName, outRec);
-        }
-      }
-    }
-    flush();
-  }
-  
-  /**
-   * Retrieves all the records managed by this MetricsContext.
-   * Useful for monitoring systems that are polling-based.
-   * @return A non-null collection of all monitoring records.
-   */
-  @Override
-  public synchronized Map<String, Collection<OutputRecord>> getAllRecords() {
-    Map<String, Collection<OutputRecord>> out = new TreeMap<String, Collection<OutputRecord>>();
-    for (String recordName : bufferedData.keySet()) {
-      RecordMap recordMap = bufferedData.get(recordName);
-      synchronized (recordMap) {
-        List<OutputRecord> records = new ArrayList<OutputRecord>();
-        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet();
-        for (Entry<TagMap, MetricMap> entry : entrySet) {
-          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
-          records.add(outRec);
-        }
-        out.put(recordName, records);
-      }
-    }
-    return out;
-  }
-
-  /**
-   * Sends a record to the metrics system.
-   */
-  protected abstract void emitRecord(String contextName, String recordName, 
-                                     OutputRecord outRec) throws IOException;
-    
-  /**
-   * Called each period after all records have been emitted, this method does nothing.
-   * Subclasses may override it in order to perform some kind of flush.
-   */
-  protected void flush() throws IOException {
-  }
-    
-  /**
-   * Called by MetricsRecordImpl.update().  Creates or updates a row in
-   * the internal table of metric data.
-   */
-  protected void update(MetricsRecordImpl record) {
-    String recordName = record.getRecordName();
-    TagMap tagTable = record.getTagTable();
-    Map<String,MetricValue> metricUpdates = record.getMetricTable();
-        
-    RecordMap recordMap = getRecordMap(recordName);
-    synchronized (recordMap) {
-      MetricMap metricMap = recordMap.get(tagTable);
-      if (metricMap == null) {
-        metricMap = new MetricMap();
-        TagMap tagMap = new TagMap(tagTable); // clone tags
-        recordMap.put(tagMap, metricMap);
-      }
-
-      Set<Entry<String, MetricValue>> entrySet = metricUpdates.entrySet();
-      for (Entry<String, MetricValue> entry : entrySet) {
-        String metricName = entry.getKey ();
-        MetricValue updateValue = entry.getValue ();
-        Number updateNumber = updateValue.getNumber();
-        Number currentNumber = metricMap.get(metricName);
-        if (currentNumber == null || updateValue.isAbsolute()) {
-          metricMap.put(metricName, updateNumber);
-        }
-        else {
-          Number newNumber = sum(updateNumber, currentNumber);
-          metricMap.put(metricName, newNumber);
-        }
-      }
-    }
-  }
-    
-  private synchronized RecordMap getRecordMap(String recordName) {
-    return bufferedData.get(recordName);
-  }
-    
-  /**
-   * Adds two numbers, coercing the second to the type of the first.
-   *
-   */
-  private Number sum(Number a, Number b) {
-    if (a instanceof Integer) {
-      return Integer.valueOf(a.intValue() + b.intValue());
-    }
-    else if (a instanceof Float) {
-      return new Float(a.floatValue() + b.floatValue());
-    }
-    else if (a instanceof Short) {
-      return Short.valueOf((short)(a.shortValue() + b.shortValue()));
-    }
-    else if (a instanceof Byte) {
-      return Byte.valueOf((byte)(a.byteValue() + b.byteValue()));
-    }
-    else if (a instanceof Long) {
-      return Long.valueOf((a.longValue() + b.longValue()));
-    }
-    else {
-      // should never happen
-      throw new MetricsException("Invalid number type");
-    }
-            
-  }
-    
-  /**
-   * Called by MetricsRecordImpl.remove().  Removes all matching rows in
-   * the internal table of metric data.  A row matches if it has the same
-   * tag names and values as record, but it may also have additional
-   * tags.
-   */    
-  protected void remove(MetricsRecordImpl record) {
-    String recordName = record.getRecordName();
-    TagMap tagTable = record.getTagTable();
-        
-    RecordMap recordMap = getRecordMap(recordName);
-    synchronized (recordMap) {
-      Iterator<TagMap> it = recordMap.keySet().iterator();
-      while (it.hasNext()) {
-        TagMap rowTags = it.next();
-        if (rowTags.containsAll(tagTable)) {
-          it.remove();
-        }
-      }
-    }
-  }
-    
-  /**
-   * Returns the timer period.
-   */
-  @Override
-  public int getPeriod() {
-    return period;
-  }
-    
-  /**
-   * Sets the timer period
-   */
-  protected void setPeriod(int period) {
-    this.period = period;
-  }
-  
-  /**
-   * If a period is set in the attribute passed in, override
-   * the default with it.
-   */
-  protected void parseAndSetPeriod(String attributeName) {
-    String periodStr = getAttribute(attributeName);
-    if (periodStr != null) {
-      int period = 0;
-      try {
-        period = Integer.parseInt(periodStr);
-      } catch (NumberFormatException nfe) {
-      }
-      if (period <= 0) {
-        throw new MetricsException("Invalid period: " + periodStr);
-      }
-      setPeriod(period);
-    }
-  }
-}
+/*
+ * AbstractMetricsContext.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics.spi;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics.ContextFactory;
+import org.apache.hadoop.metrics.MetricsContext;
+import org.apache.hadoop.metrics.MetricsException;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.Updater;
+
+/**
+ * The main class of the Service Provider Interface.  This class should be
+ * extended in order to integrate the Metrics API with a specific metrics
+ * client library. <p/>
+ *
+ * This class implements the internal table of metric data, and the timer
+ * on which data is to be sent to the metrics system.  Subclasses must
+ * override the abstract <code>emitRecord</code> method in order to transmit
+ * the data. <p/>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class AbstractMetricsContext implements MetricsContext {
+    
+  private int period = MetricsContext.DEFAULT_PERIOD;
+  private Timer timer = null;
+    
+  private Set<Updater> updaters = new HashSet<Updater>(1);
+  private volatile boolean isMonitoring = false;
+    
+  private ContextFactory factory = null;
+  private String contextName = null;
+    
+  @InterfaceAudience.Private
+  public static class TagMap extends TreeMap<String,Object> {
+    private static final long serialVersionUID = 3546309335061952993L;
+    TagMap() {
+      super();
+    }
+    TagMap(TagMap orig) {
+      super(orig);
+    }
+    /**
+     * Returns true if this tagmap contains every tag in other.
+     */
+    public boolean containsAll(TagMap other) {
+      for (Map.Entry<String,Object> entry : other.entrySet()) {
+        Object value = get(entry.getKey());
+        if (value == null || !value.equals(entry.getValue())) {
+          // either key does not exist here, or the value is different
+          return false;
+        }
+      }
+      return true;
+    }
+  }
+  
+  @InterfaceAudience.Private
+  public static class MetricMap extends TreeMap<String,Number> {
+    private static final long serialVersionUID = -7495051861141631609L;
+    MetricMap() {
+      super();
+    }
+    MetricMap(MetricMap orig) {
+      super(orig);
+    }
+  }
+            
+  static class RecordMap extends HashMap<TagMap,MetricMap> {
+    private static final long serialVersionUID = 259835619700264611L;
+  }
+    
+  private Map<String,RecordMap> bufferedData = new HashMap<String,RecordMap>();
+    
+
+  /**
+   * Creates a new instance of AbstractMetricsContext
+   */
+  protected AbstractMetricsContext() {
+  }
+    
+  /**
+   * Initializes the context.
+   */
+  @Override
+  public void init(String contextName, ContextFactory factory) 
+  {
+    this.contextName = contextName;
+    this.factory = factory;
+  }
+    
+  /**
+   * Convenience method for subclasses to access factory attributes.
+   */
+  protected String getAttribute(String attributeName) {
+    String factoryAttribute = contextName + "." + attributeName;
+    return (String) factory.getAttribute(factoryAttribute);  
+  }
+    
+  /**
+   * Returns an attribute-value map derived from the factory attributes
+   * by finding all factory attributes that begin with 
+   * <i>contextName</i>.<i>tableName</i>.  The returned map consists of
+   * those attributes with the contextName and tableName stripped off.
+   */
+  protected Map<String,String> getAttributeTable(String tableName) {
+    String prefix = contextName + "." + tableName + ".";
+    Map<String,String> result = new HashMap<String,String>();
+    for (String attributeName : factory.getAttributeNames()) {
+      if (attributeName.startsWith(prefix)) {
+        String name = attributeName.substring(prefix.length());
+        String value = (String) factory.getAttribute(attributeName);
+        result.put(name, value);
+      }
+    }
+    return result;
+  }
+    
+  /**
+   * Returns the context name.
+   */
+  @Override
+  public String getContextName() {
+    return contextName;
+  }
+    
+  /**
+   * Returns the factory by which this context was created.
+   */
+  public ContextFactory getContextFactory() {
+    return factory;
+  }
+    
+  /**
+   * Starts or restarts monitoring, the emitting of metrics records.
+   */
+  @Override
+  public synchronized void startMonitoring()
+    throws IOException {
+    if (!isMonitoring) {
+      startTimer();
+      isMonitoring = true;
+    }
+  }
+    
+  /**
+   * Stops monitoring.  This does not free buffered data. 
+   * @see #close()
+   */
+  @Override
+  public synchronized void stopMonitoring() {
+    if (isMonitoring) {
+      stopTimer();
+      isMonitoring = false;
+    }
+  }
+    
+  /**
+   * Returns true if monitoring is currently in progress.
+   */
+  @Override
+  public boolean isMonitoring() {
+    return isMonitoring;
+  }
+    
+  /**
+   * Stops monitoring and frees buffered data, returning this
+   * object to its initial state.  
+   */
+  @Override
+  public synchronized void close() {
+    stopMonitoring();
+    clearUpdaters();
+  } 
+    
+  /**
+   * Creates a new AbstractMetricsRecord instance with the given <code>recordName</code>.
+   * Throws an exception if the metrics implementation is configured with a fixed
+   * set of record names and <code>recordName</code> is not in that set.
+   * 
+   * @param recordName the name of the record
+   * @throws MetricsException if recordName conflicts with configuration data
+   */
+  @Override
+  public final synchronized MetricsRecord createRecord(String recordName) {
+    if (bufferedData.get(recordName) == null) {
+      bufferedData.put(recordName, new RecordMap());
+    }
+    return newRecord(recordName);
+  }
+    
+  /**
+   * Subclasses should override this if they subclass MetricsRecordImpl.
+   * @param recordName the name of the record
+   * @return newly created instance of MetricsRecordImpl or subclass
+   */
+  protected MetricsRecord newRecord(String recordName) {
+    return new MetricsRecordImpl(recordName, this);
+  }
+    
+  /**
+   * Registers a callback to be called at time intervals determined by
+   * the configuration.
+   *
+   * @param updater object to be run periodically; it should update
+   * some metrics records 
+   */
+  @Override
+  public synchronized void registerUpdater(final Updater updater) {
+    if (!updaters.contains(updater)) {
+      updaters.add(updater);
+    }
+  }
+    
+  /**
+   * Removes a callback, if it exists.
+   *
+   * @param updater object to be removed from the callback list
+   */
+  @Override
+  public synchronized void unregisterUpdater(Updater updater) {
+    updaters.remove(updater);
+  }
+    
+  private synchronized void clearUpdaters() {
+    updaters.clear();
+  }
+    
+  /**
+   * Starts timer if it is not already started
+   */
+  private synchronized void startTimer() {
+    if (timer == null) {
+      timer = new Timer("Timer thread for monitoring " + getContextName(), 
+                        true);
+      TimerTask task = new TimerTask() {
+          @Override
+          public void run() {
+            try {
+              timerEvent();
+            } catch (IOException ioe) {
+              ioe.printStackTrace();
+            }
+          }
+        };
+      long millis = period * 1000;
+      timer.scheduleAtFixedRate(task, millis, millis);
+    }
+  }
+    
+  /**
+   * Stops timer if it is running
+   */
+  private synchronized void stopTimer() {
+    if (timer != null) {
+      timer.cancel();
+      timer = null;
+    }
+  }
+    
+  /**
+   * Timer callback.
+   */
+  private void timerEvent() throws IOException {
+    if (isMonitoring) {
+      Collection<Updater> myUpdaters;
+      synchronized (this) {
+        myUpdaters = new ArrayList<Updater>(updaters);
+      }     
+      // Run all the registered updates without holding a lock
+      // on this context
+      for (Updater updater : myUpdaters) {
+        try {
+          updater.doUpdates(this);
+        } catch (Throwable throwable) {
+          throwable.printStackTrace();
+        }
+      }
+      emitRecords();
+    }
+  }
+    
+  /**
+   *  Emits the records.
+   */
+  private synchronized void emitRecords() throws IOException {
+    for (String recordName : bufferedData.keySet()) {
+      RecordMap recordMap = bufferedData.get(recordName);
+      synchronized (recordMap) {
+        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet ();
+        for (Entry<TagMap, MetricMap> entry : entrySet) {
+          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
+          emitRecord(contextName, recordName, outRec);
+        }
+      }
+    }
+    flush();
+  }
+  
+  /**
+   * Retrieves all the records managed by this MetricsContext.
+   * Useful for monitoring systems that are polling-based.
+   * @return A non-null collection of all monitoring records.
+   */
+  @Override
+  public synchronized Map<String, Collection<OutputRecord>> getAllRecords() {
+    Map<String, Collection<OutputRecord>> out = new TreeMap<String, Collection<OutputRecord>>();
+    for (String recordName : bufferedData.keySet()) {
+      RecordMap recordMap = bufferedData.get(recordName);
+      synchronized (recordMap) {
+        List<OutputRecord> records = new ArrayList<OutputRecord>();
+        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet();
+        for (Entry<TagMap, MetricMap> entry : entrySet) {
+          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
+          records.add(outRec);
+        }
+        out.put(recordName, records);
+      }
+    }
+    return out;
+  }
+
+  /**
+   * Sends a record to the metrics system.
+   */
+  protected abstract void emitRecord(String contextName, String recordName, 
+                                     OutputRecord outRec) throws IOException;
+    
+  /**
+   * Called each period after all records have been emitted, this method does nothing.
+   * Subclasses may override it in order to perform some kind of flush.
+   */
+  protected void flush() throws IOException {
+  }
+    
+  /**
+   * Called by MetricsRecordImpl.update().  Creates or updates a row in
+   * the internal table of metric data.
+   */
+  protected void update(MetricsRecordImpl record) {
+    String recordName = record.getRecordName();
+    TagMap tagTable = record.getTagTable();
+    Map<String,MetricValue> metricUpdates = record.getMetricTable();
+        
+    RecordMap recordMap = getRecordMap(recordName);
+    synchronized (recordMap) {
+      MetricMap metricMap = recordMap.get(tagTable);
+      if (metricMap == null) {
+        metricMap = new MetricMap();
+        TagMap tagMap = new TagMap(tagTable); // clone tags
+        recordMap.put(tagMap, metricMap);
+      }
+
+      Set<Entry<String, MetricValue>> entrySet = metricUpdates.entrySet();
+      for (Entry<String, MetricValue> entry : entrySet) {
+        String metricName = entry.getKey ();
+        MetricValue updateValue = entry.getValue ();
+        Number updateNumber = updateValue.getNumber();
+        Number currentNumber = metricMap.get(metricName);
+        if (currentNumber == null || updateValue.isAbsolute()) {
+          metricMap.put(metricName, updateNumber);
+        }
+        else {
+          Number newNumber = sum(updateNumber, currentNumber);
+          metricMap.put(metricName, newNumber);
+        }
+      }
+    }
+  }
+    
+  private synchronized RecordMap getRecordMap(String recordName) {
+    return bufferedData.get(recordName);
+  }
+    
+  /**
+   * Adds two numbers, coercing the second to the type of the first.
+   *
+   */
+  private Number sum(Number a, Number b) {
+    if (a instanceof Integer) {
+      return Integer.valueOf(a.intValue() + b.intValue());
+    }
+    else if (a instanceof Float) {
+      return new Float(a.floatValue() + b.floatValue());
+    }
+    else if (a instanceof Short) {
+      return Short.valueOf((short)(a.shortValue() + b.shortValue()));
+    }
+    else if (a instanceof Byte) {
+      return Byte.valueOf((byte)(a.byteValue() + b.byteValue()));
+    }
+    else if (a instanceof Long) {
+      return Long.valueOf((a.longValue() + b.longValue()));
+    }
+    else {
+      // should never happen
+      throw new MetricsException("Invalid number type");
+    }
+            
+  }
+    
+  /**
+   * Called by MetricsRecordImpl.remove().  Removes all matching rows in
+   * the internal table of metric data.  A row matches if it has the same
+   * tag names and values as record, but it may also have additional
+   * tags.
+   */    
+  protected void remove(MetricsRecordImpl record) {
+    String recordName = record.getRecordName();
+    TagMap tagTable = record.getTagTable();
+        
+    RecordMap recordMap = getRecordMap(recordName);
+    synchronized (recordMap) {
+      Iterator<TagMap> it = recordMap.keySet().iterator();
+      while (it.hasNext()) {
+        TagMap rowTags = it.next();
+        if (rowTags.containsAll(tagTable)) {
+          it.remove();
+        }
+      }
+    }
+  }
+    
+  /**
+   * Returns the timer period.
+   */
+  @Override
+  public int getPeriod() {
+    return period;
+  }
+    
+  /**
+   * Sets the timer period
+   */
+  protected void setPeriod(int period) {
+    this.period = period;
+  }
+  
+  /**
+   * If a period is set in the attribute passed in, override
+   * the default with it.
+   */
+  protected void parseAndSetPeriod(String attributeName) {
+    String periodStr = getAttribute(attributeName);
+    if (periodStr != null) {
+      int period = 0;
+      try {
+        period = Integer.parseInt(periodStr);
+      } catch (NumberFormatException nfe) {
+      }
+      if (period <= 0) {
+        throw new MetricsException("Invalid period: " + periodStr);
+      }
+      setPeriod(period);
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java

@@ -1,300 +1,300 @@
-/*
- * MetricsRecordImpl.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics.spi;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.MetricsException;
-import org.apache.hadoop.metrics.MetricsRecord;
-import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap;
-
-/**
- * An implementation of MetricsRecord.  Keeps a back-pointer to the context
- * from which it was created, and delegates back to it on <code>update</code>
- * and <code>remove()</code>.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class MetricsRecordImpl implements MetricsRecord {
-    
-  private TagMap tagTable = new TagMap();
-  private Map<String,MetricValue> metricTable = new LinkedHashMap<String,MetricValue>();
-    
-  private String recordName;
-  private AbstractMetricsContext context;
-    
-    
-  /** Creates a new instance of FileRecord */
-  protected MetricsRecordImpl(String recordName, AbstractMetricsContext context)
-  {
-    this.recordName = recordName;
-    this.context = context;
-  }
-    
-  /**
-   * Returns the record name. 
-   *
-   * @return the record name
-   */
-  @Override
-  public String getRecordName() {
-    return recordName;
-  }
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  @Override
-  public void setTag(String tagName, String tagValue) {
-    if (tagValue == null) {
-      tagValue = "";
-    }
-    tagTable.put(tagName, tagValue);
-  }
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  @Override
-  public void setTag(String tagName, int tagValue) {
-    tagTable.put(tagName, Integer.valueOf(tagValue));
-  }
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  @Override
-  public void setTag(String tagName, long tagValue) {
-    tagTable.put(tagName, Long.valueOf(tagValue));
-  }
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  @Override
-  public void setTag(String tagName, short tagValue) {
-    tagTable.put(tagName, Short.valueOf(tagValue));
-  }
-    
-  /**
-   * Sets the named tag to the specified value.
-   *
-   * @param tagName name of the tag
-   * @param tagValue new value of the tag
-   * @throws MetricsException if the tagName conflicts with the configuration
-   */
-  @Override
-  public void setTag(String tagName, byte tagValue) {
-    tagTable.put(tagName, Byte.valueOf(tagValue));
-  }
-    
-  /**
-   * Removes any tag of the specified name.
-   */
-  @Override
-  public void removeTag(String tagName) {
-    tagTable.remove(tagName);
-  }
-  
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void setMetric(String metricName, int metricValue) {
-    setAbsolute(metricName, Integer.valueOf(metricValue));
-  }
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void setMetric(String metricName, long metricValue) {
-    setAbsolute(metricName, Long.valueOf(metricValue));
-  }
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void setMetric(String metricName, short metricValue) {
-    setAbsolute(metricName, Short.valueOf(metricValue));
-  }
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void setMetric(String metricName, byte metricValue) {
-    setAbsolute(metricName, Byte.valueOf(metricValue));
-  }
-    
-  /**
-   * Sets the named metric to the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue new value of the metric
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void setMetric(String metricName, float metricValue) {
-    setAbsolute(metricName, new Float(metricValue));
-  }
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void incrMetric(String metricName, int metricValue) {
-    setIncrement(metricName, Integer.valueOf(metricValue));
-  }
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void incrMetric(String metricName, long metricValue) {
-    setIncrement(metricName, Long.valueOf(metricValue));
-  }
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void incrMetric(String metricName, short metricValue) {
-    setIncrement(metricName, Short.valueOf(metricValue));
-  }
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void incrMetric(String metricName, byte metricValue) {
-    setIncrement(metricName, Byte.valueOf(metricValue));
-  }
-    
-  /**
-   * Increments the named metric by the specified value.
-   *
-   * @param metricName name of the metric
-   * @param metricValue incremental value
-   * @throws MetricsException if the metricName or the type of the metricValue 
-   * conflicts with the configuration
-   */
-  @Override
-  public void incrMetric(String metricName, float metricValue) {
-    setIncrement(metricName, new Float(metricValue));
-  }
-    
-  private void setAbsolute(String metricName, Number metricValue) {
-    metricTable.put(metricName, new MetricValue(metricValue, MetricValue.ABSOLUTE));
-  }
-    
-  private void setIncrement(String metricName, Number metricValue) {
-    metricTable.put(metricName, new MetricValue(metricValue, MetricValue.INCREMENT));
-  }
-    
-  /**
-   * Updates the table of buffered data which is to be sent periodically.
-   * If the tag values match an existing row, that row is updated; 
-   * otherwise, a new row is added.
-   */
-  @Override
-  public void update() {
-    context.update(this);
-  }
-    
-  /**
-   * Removes the row, if it exists, in the buffered data table having tags 
-   * that equal the tags that have been set on this record. 
-   */
-  @Override
-  public void remove() {
-    context.remove(this);
-  }
-
-  TagMap getTagTable() {
-    return tagTable;
-  }
-
-  Map<String, MetricValue> getMetricTable() {
-    return metricTable;
-  }
-}
+/*
+ * MetricsRecordImpl.java
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics.spi;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics.MetricsException;
+import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap;
+
+/**
+ * An implementation of MetricsRecord.  Keeps a back-pointer to the context
+ * from which it was created, and delegates back to it on <code>update</code>
+ * and <code>remove()</code>.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class MetricsRecordImpl implements MetricsRecord {
+    
+  private TagMap tagTable = new TagMap();
+  private Map<String,MetricValue> metricTable = new LinkedHashMap<String,MetricValue>();
+    
+  private String recordName;
+  private AbstractMetricsContext context;
+    
+    
+  /** Creates a new instance of FileRecord */
+  protected MetricsRecordImpl(String recordName, AbstractMetricsContext context)
+  {
+    this.recordName = recordName;
+    this.context = context;
+  }
+    
+  /**
+   * Returns the record name. 
+   *
+   * @return the record name
+   */
+  @Override
+  public String getRecordName() {
+    return recordName;
+  }
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  @Override
+  public void setTag(String tagName, String tagValue) {
+    if (tagValue == null) {
+      tagValue = "";
+    }
+    tagTable.put(tagName, tagValue);
+  }
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  @Override
+  public void setTag(String tagName, int tagValue) {
+    tagTable.put(tagName, Integer.valueOf(tagValue));
+  }
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  @Override
+  public void setTag(String tagName, long tagValue) {
+    tagTable.put(tagName, Long.valueOf(tagValue));
+  }
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  @Override
+  public void setTag(String tagName, short tagValue) {
+    tagTable.put(tagName, Short.valueOf(tagValue));
+  }
+    
+  /**
+   * Sets the named tag to the specified value.
+   *
+   * @param tagName name of the tag
+   * @param tagValue new value of the tag
+   * @throws MetricsException if the tagName conflicts with the configuration
+   */
+  @Override
+  public void setTag(String tagName, byte tagValue) {
+    tagTable.put(tagName, Byte.valueOf(tagValue));
+  }
+    
+  /**
+   * Removes any tag of the specified name.
+   */
+  @Override
+  public void removeTag(String tagName) {
+    tagTable.remove(tagName);
+  }
+  
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void setMetric(String metricName, int metricValue) {
+    setAbsolute(metricName, Integer.valueOf(metricValue));
+  }
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void setMetric(String metricName, long metricValue) {
+    setAbsolute(metricName, Long.valueOf(metricValue));
+  }
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void setMetric(String metricName, short metricValue) {
+    setAbsolute(metricName, Short.valueOf(metricValue));
+  }
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void setMetric(String metricName, byte metricValue) {
+    setAbsolute(metricName, Byte.valueOf(metricValue));
+  }
+    
+  /**
+   * Sets the named metric to the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue new value of the metric
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void setMetric(String metricName, float metricValue) {
+    setAbsolute(metricName, new Float(metricValue));
+  }
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void incrMetric(String metricName, int metricValue) {
+    setIncrement(metricName, Integer.valueOf(metricValue));
+  }
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void incrMetric(String metricName, long metricValue) {
+    setIncrement(metricName, Long.valueOf(metricValue));
+  }
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void incrMetric(String metricName, short metricValue) {
+    setIncrement(metricName, Short.valueOf(metricValue));
+  }
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void incrMetric(String metricName, byte metricValue) {
+    setIncrement(metricName, Byte.valueOf(metricValue));
+  }
+    
+  /**
+   * Increments the named metric by the specified value.
+   *
+   * @param metricName name of the metric
+   * @param metricValue incremental value
+   * @throws MetricsException if the metricName or the type of the metricValue 
+   * conflicts with the configuration
+   */
+  @Override
+  public void incrMetric(String metricName, float metricValue) {
+    setIncrement(metricName, new Float(metricValue));
+  }
+    
+  private void setAbsolute(String metricName, Number metricValue) {
+    metricTable.put(metricName, new MetricValue(metricValue, MetricValue.ABSOLUTE));
+  }
+    
+  private void setIncrement(String metricName, Number metricValue) {
+    metricTable.put(metricName, new MetricValue(metricValue, MetricValue.INCREMENT));
+  }
+    
+  /**
+   * Updates the table of buffered data which is to be sent periodically.
+   * If the tag values match an existing row, that row is updated; 
+   * otherwise, a new row is added.
+   */
+  @Override
+  public void update() {
+    context.update(this);
+  }
+    
+  /**
+   * Removes the row, if it exists, in the buffered data table having tags 
+   * that equal the tags that have been set on this record. 
+   */
+  @Override
+  public void remove() {
+    context.remove(this);
+  }
+
+  TagMap getTagTable() {
+    return tagTable;
+  }
+
+  Map<String, MetricValue> getMetricTable() {
+    return metricTable;
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java

@@ -30,14 +30,14 @@ public interface MetricsCollector {
   /**
    * Add a metrics record
    * @param name  of the record
-   * @return  a metrics record builder for the record
+   * @return  a {@link MetricsRecordBuilder} for the record {@code name}
    */
   public MetricsRecordBuilder addRecord(String name);
 
   /**
    * Add a metrics record
    * @param info  of the record
-   * @return  a metrics record builder for the record
+   * @return  a {@link MetricsRecordBuilder} for metrics {@code info}
    */
   public MetricsRecordBuilder addRecord(MetricsInfo info);
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java

@@ -22,7 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics filter interface
+ * The metrics filter interface. The MetricsFilter objects can be used either to
+ * filter the metrics from {@link MetricsSource}s or to filter metrics per
+ * {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java

@@ -20,19 +20,24 @@ package org.apache.hadoop.metrics2;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.annotation.Metric;
 
 /**
- * Interface to provide immutable meta info for metrics
+ * Interface to provide immutable metainfo for metrics.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface MetricsInfo {
   /**
+   * Typically name corresponds to annotation {@link Metric#value()} or
+   * the name of the class.
    * @return the name of the metric/tag
    */
   String name();
 
   /**
+   * Typically the description corresponds to annotation {@link Metric#about()}
+   * or the name of the class.
    * @return the description of the metric/tag
    */
   String description();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java

@@ -36,17 +36,17 @@ public interface MetricsRecord {
   long timestamp();
 
   /**
-   * @return the record name
+   * @return the metrics record name
    */
   String name();
 
   /**
-   * @return the description of the record
+   * @return the description of the metrics record
    */
   String description();
 
   /**
-   * @return the context name of the record
+   * @return the context name of the metrics record
    */
   String context();
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceStability.Evolving
 public abstract class MetricsRecordBuilder {
   /**
-   * Add a metrics tag
+   * Add a metrics value with metrics information
    * @param info  metadata of the tag
    * @param value of the tag
    * @return self

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java

@@ -22,7 +22,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics sink interface
+ * The metrics sink interface. <p>
+ * Implementations of this interface consume the {@link MetricsRecord} generated
+ * from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ * periodically pushes the {@link MetricsRecord} to the sink using
+ * {@link #putMetrics(MetricsRecord)} method.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java

@@ -22,13 +22,15 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics source interface
+ * The source of metrics information. It generates and updates metrics. It
+ * registers with {@link MetricsSystem}, which periodically polls it to collect
+ * {@link MetricsRecord} and passes it to {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface MetricsSource {
   /**
-   * Get metrics from the source
+   * Get metrics from the metrics source
    * @param collector to contain the resulting metrics snapshot
    * @param all if true, return all metrics even if unchanged.
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java

@@ -22,7 +22,18 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics system interface
+ * The metrics system interface.
+ * 
+ * The following components are used for metrics.
+ * <ul>
+ * <li>{@link MetricsSource} generate and update metrics information.</li>
+ * <li>{@link MetricsSink} consume the metrics information</li>
+ * </ul>
+ * 
+ * {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ * system. Implementations of {@link MetricsSystem} polls the
+ * {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ * {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java

@@ -24,7 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * Annotation interface for a single metric
+ * Annotation interface for a single metric used to annotate a field or a method
+ * in the class.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java

@@ -32,6 +32,14 @@ import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.lib.Interns;
 import org.apache.hadoop.util.Time;
 
+/**
+ * {@link MetricsRecordBuilder} implementation used for building metrics records
+ * by the {@link MetricsCollector}. It provides the following functionality:
+ * <ul>
+ * <li>Allows configuring filters for metrics.
+ * </ul>
+ *
+ */
 class MetricsRecordBuilderImpl extends MetricsRecordBuilder {
   private final MetricsCollector parent;
   private final long timestamp;
@@ -41,9 +49,15 @@ class MetricsRecordBuilderImpl extends MetricsRecordBuilder {
   private final MetricsFilter recordFilter, metricFilter;
   private final boolean acceptable;
 
+  /**
+   * @param parent {@link MetricsCollector} using this record builder
+   * @param info metrics information
+   * @param rf
+   * @param mf
+   * @param acceptable
+   */
   MetricsRecordBuilderImpl(MetricsCollector parent, MetricsInfo info,
-                           MetricsFilter rf, MetricsFilter mf,
-                           boolean acceptable) {
+      MetricsFilter rf, MetricsFilter mf, boolean acceptable) {
     this.parent = parent;
     timestamp = Time.now();
     recInfo = info;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java

@@ -37,7 +37,7 @@ class MetricsRecordImpl extends AbstractMetricsRecord {
 
   /**
    * Construct a metrics record
-   * @param info  {@link MetricInfo} of the record
+   * @param info  {@link MetricsInfo} of the record
    * @param timestamp of the record
    * @param tags  of the record
    * @param metrics of the record

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java

@@ -235,11 +235,9 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
   void registerSource(String name, String desc, MetricsSource source) {
     checkNotNull(config, "config");
     MetricsConfig conf = sourceConfigs.get(name);
-    MetricsSourceAdapter sa = conf != null
-        ? new MetricsSourceAdapter(prefix, name, desc, source,
-                                   injectedTags, period, conf)
-        : new MetricsSourceAdapter(prefix, name, desc, source,
-          injectedTags, period, config.subset(SOURCE_KEY));
+    MetricsSourceAdapter sa = new MetricsSourceAdapter(prefix, name, desc,
+        source, injectedTags, period, conf != null ? conf
+            : config.subset(SOURCE_KEY));
     sources.put(name, sa);
     sa.start();
     LOG.debug("Registered source "+ name);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java

@@ -27,8 +27,13 @@ import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
- * The default metrics system singleton
+ * The default metrics system singleton. This class is used by all the daemon
+ * processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ * initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ * to initialize the {@link MetricsSystem}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
@@ -37,7 +42,10 @@ public enum DefaultMetricsSystem {
 
   private AtomicReference<MetricsSystem> impl =
       new AtomicReference<MetricsSystem>(new MetricsSystemImpl());
+  
+  @VisibleForTesting
   volatile boolean miniClusterMode = false;
+  
   final UniqueNames mBeanNames = new UniqueNames();
   final UniqueNames sourceNames = new UniqueNames();
 
@@ -87,12 +95,12 @@ public enum DefaultMetricsSystem {
 
   MetricsSystem getImpl() { return impl.get(); }
 
-  @InterfaceAudience.Private
+  @VisibleForTesting
   public static void setMiniClusterMode(boolean choice) {
     INSTANCE.miniClusterMode = choice;
   }
 
-  @InterfaceAudience.Private
+  @VisibleForTesting
   public static boolean inMiniClusterMode() {
     return INSTANCE.miniClusterMode;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java

@@ -35,7 +35,16 @@ import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 
 /**
- * Helper class to build metrics source object from annotations
+ * Helper class to build {@link MetricsSource} object from annotations.
+ * <p>
+ * For a given source object:
+ * <ul>
+ * <li>Sets the {@link Field}s annotated with {@link Metric} to
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * <li>
+ * For {@link Method}s annotated with {@link Metric} creates
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * </ul>
  */
 @InterfaceAudience.Private
 public class MetricsSourceBuilder {
@@ -115,9 +124,15 @@ public class MetricsSourceBuilder {
     return r;
   }
 
+  /**
+   * Change the declared field {@code field} in {@code source} Object to
+   * {@link MutableMetric}
+   */
   private void add(Object source, Field field) {
     for (Annotation annotation : field.getAnnotations()) {
-      if (!(annotation instanceof Metric)) continue;
+      if (!(annotation instanceof Metric)) {
+        continue;
+      }
       try {
         // skip fields already set
         field.setAccessible(true);
@@ -131,7 +146,7 @@ public class MetricsSourceBuilder {
                                                   registry);
       if (mutable != null) {
         try {
-          field.set(source, mutable);
+          field.set(source, mutable); // Set the source field to MutableMetric
           hasAtMetric = true;
         } catch (Exception e) {
           throw new MetricsException("Error setting field "+ field +
@@ -141,9 +156,12 @@ public class MetricsSourceBuilder {
     }
   }
 
+  /** Add {@link MutableMetric} for a method annotated with {@link Metric} */
   private void add(Object source, Method method) {
     for (Annotation annotation : method.getAnnotations()) {
-      if (!(annotation instanceof Metric)) continue;
+      if (!(annotation instanceof Metric)) {
+        continue;
+      }
       factory.newForMethod(source, method, (Metric) annotation, registry);
       hasAtMetric = true;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java

@@ -130,6 +130,10 @@ public class MutableMetricsFactory {
     return Interns.info(name2, about.isEmpty() ? name2 : about);
   }
 
+  /**
+   * Remove the prefix "get", if any, from the method name. Return the
+   * capacitalized method name."
+   */
   protected String getName(Method method) {
     String methodName = method.getName();
     if (methodName.startsWith("get")) {
@@ -140,12 +144,15 @@ public class MutableMetricsFactory {
 
   protected MetricsInfo getInfo(Metric annotation, String defaultName) {
     String[] value = annotation.value();
-     if (value.length == 2) {
+    if (value.length == 2) {
+      // Use name and description from the annotation
       return Interns.info(value[0], value[1]);
     }
     if (value.length == 1) {
+      // Use description from the annotation and method name as metric name
       return Interns.info(defaultName, value[0]);
     }
+    // Use method name as metric name and description
     return Interns.info(defaultName, defaultName);
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.lib;
 
 import static org.apache.hadoop.metrics2.lib.Interns.info;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -150,14 +149,8 @@ public class MutableQuantiles extends MutableMetric {
     @Override
     public void run() {
       synchronized (parent) {
-        try {
-          parent.previousCount = parent.estimator.getCount();
-          parent.previousSnapshot = parent.estimator.snapshot();
-        } catch (IOException e) {
-          // Couldn't get a new snapshot because the window was empty
-          parent.previousCount = 0;
-          parent.previousSnapshot = null;
-        }
+        parent.previousCount = parent.estimator.getCount();
+        parent.previousSnapshot = parent.estimator.snapshot();
         parent.estimator.clear();
       }
       parent.setChanged();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java

@@ -20,12 +20,14 @@ package org.apache.hadoop.metrics2.util;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 
+import com.google.common.collect.ComparisonChain;
+
 /**
  * Specifies a quantile (with error bounds) to be watched by a
  * {@link SampleQuantiles} object.
  */
 @InterfaceAudience.Private
-public class Quantile {
+public class Quantile implements Comparable<Quantile> {
   public final double quantile;
   public final double error;
 
@@ -57,4 +59,19 @@ public class Quantile {
     return (int) (Double.doubleToLongBits(quantile) ^ Double
         .doubleToLongBits(error));
   }
+
+  @Override
+  public int compareTo(Quantile other) {
+    return ComparisonChain.start()
+        .compare(quantile, other.quantile)
+        .compare(error, other.error)
+        .result();
+  }
+  
+  @Override
+  public String toString() {
+    return String.format("%.2f %%ile +/- %.2f%%",
+        quantile * 100, error * 100);
+  }
+
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java

@@ -18,16 +18,17 @@
 
 package org.apache.hadoop.metrics2.util;
 
-import java.io.IOException;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 
 /**
  * Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm
@@ -202,10 +203,8 @@ public class SampleQuantiles {
    * @param quantile Queried quantile, e.g. 0.50 or 0.99.
    * @return Estimated value at that quantile.
    */
-  private long query(double quantile) throws IOException {
-    if (samples.size() == 0) {
-      throw new IOException("No samples present");
-    }
+  private long query(double quantile) {
+    Preconditions.checkState(!samples.isEmpty(), "no data in estimator");
 
     int rankMin = 0;
     int desired = (int) (quantile * count);
@@ -231,14 +230,18 @@ public class SampleQuantiles {
   /**
    * Get a snapshot of the current values of all the tracked quantiles.
    * 
-   * @return snapshot of the tracked quantiles
-   * @throws IOException
-   *           if no items have been added to the estimator
+   * @return snapshot of the tracked quantiles. If no items are added
+   * to the estimator, returns null.
    */
-  synchronized public Map<Quantile, Long> snapshot() throws IOException {
+  synchronized public Map<Quantile, Long> snapshot() {
     // flush the buffer first for best results
     insertBatch();
-    Map<Quantile, Long> values = new HashMap<Quantile, Long>(quantiles.length);
+    
+    if (samples.isEmpty()) {
+      return null;
+    }
+    
+    Map<Quantile, Long> values = new TreeMap<Quantile, Long>();
     for (int i = 0; i < quantiles.length; i++) {
       values.put(quantiles[i], query(quantiles[i].quantile));
     }
@@ -273,6 +276,16 @@ public class SampleQuantiles {
     bufferCount = 0;
     samples.clear();
   }
+  
+  @Override
+  synchronized public String toString() {
+    Map<Quantile, Long> data = snapshot();
+    if (data == null) {
+      return "[no samples]";
+    } else {
+      return Joiner.on("\n").withKeyValueSeparator(": ").join(data);
+    }
+  }
 
   /**
    * Describes a measured value passed to the estimator, tracking additional

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java

@@ -48,7 +48,7 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
       throw new RuntimeException("Bailing out since native library couldn't " +
         "be loaded");
     }
-    LOG.info("Using JniBasedUnixGroupsMapping for Group resolution");
+    LOG.debug("Using JniBasedUnixGroupsMapping for Group resolution");
   }
 
   @Override
@@ -57,7 +57,11 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
     try {
       groups = getGroupForUser(user);
     } catch (Exception e) {
-      LOG.warn("Error getting groups for " + user, e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error getting groups for " + user, e);
+      } else {
+        LOG.info("Error getting groups for " + user + ": " + e.getMessage());
+      }
     }
     return Arrays.asList(groups);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java

@@ -52,7 +52,7 @@ public class JniBasedUnixGroupsNetgroupMapping
       throw new RuntimeException("Bailing out since native library couldn't " +
         "be loaded");
     }
-    LOG.info("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
+    LOG.debug("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
   }
 
   /**
@@ -115,7 +115,12 @@ public class JniBasedUnixGroupsNetgroupMapping
       // JNI code does not expect '@' at the begining of the group name
       users = getUsersForNetgroupJNI(netgroup.substring(1));
     } catch (Exception e) {
-      LOG.warn("error getting users for netgroup " + netgroup, e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error getting users for netgroup " + netgroup, e);
+      } else {
+        LOG.info("Error getting users for netgroup " + netgroup + 
+            ": " + e.getMessage());
+      }
     }
     if (users != null && users.length != 0) {
       return Arrays.asList(users);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java

@@ -1,460 +1,460 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumException;
-
-/**
- * This class provides inteface and utilities for processing checksums for
- * DFS data transfers.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Evolving
-public class DataChecksum implements Checksum {
-  
-  // Misc constants
-  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
-  
-  // checksum types
-  public static final int CHECKSUM_NULL    = 0;
-  public static final int CHECKSUM_CRC32   = 1;
-  public static final int CHECKSUM_CRC32C  = 2;
-  public static final int CHECKSUM_DEFAULT = 3; 
-  public static final int CHECKSUM_MIXED   = 4;
- 
-  /** The checksum types */
-  public static enum Type {
-    NULL  (CHECKSUM_NULL, 0),
-    CRC32 (CHECKSUM_CRC32, 4),
-    CRC32C(CHECKSUM_CRC32C, 4),
-    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
-    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
-
-    public final int id;
-    public final int size;
-    
-    private Type(int id, int size) {
-      this.id = id;
-      this.size = size;
-    }
-
-    /** @return the type corresponding to the id. */
-    public static Type valueOf(int id) {
-      if (id < 0 || id >= values().length) {
-        throw new IllegalArgumentException("id=" + id
-            + " out of range [0, " + values().length + ")");
-      }
-      return values()[id];
-    }
-  }
-
-
-  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
-    if ( bytesPerChecksum <= 0 ) {
-      return null;
-    }
-    
-    switch ( type ) {
-    case NULL :
-      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
-    case CRC32 :
-      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
-    case CRC32C:
-      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
-    default:
-      return null;  
-    }
-  }
-  
-  /**
-   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
-   * @return DataChecksum of the type in the array or null in case of an error.
-   */
-  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
-    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
-      return null;
-    }
-    
-    // like readInt():
-    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
-                           ( (bytes[offset+2] & 0xff) << 16 ) |
-                           ( (bytes[offset+3] & 0xff) << 8 )  |
-                           ( (bytes[offset+4] & 0xff) );
-    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
-  }
-  
-  /**
-   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
-   * input stream <i>in</i>
-   */
-  public static DataChecksum newDataChecksum( DataInputStream in )
-                                 throws IOException {
-    int type = in.readByte();
-    int bpc = in.readInt();
-    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
-    if ( summer == null ) {
-      throw new IOException( "Could not create DataChecksum of type " +
-                             type + " with bytesPerChecksum " + bpc );
-    }
-    return summer;
-  }
-  
-  /**
-   * Writes the checksum header to the output stream <i>out</i>.
-   */
-  public void writeHeader( DataOutputStream out ) 
-                           throws IOException { 
-    out.writeByte( type.id );
-    out.writeInt( bytesPerChecksum );
-  }
-
-  public byte[] getHeader() {
-    byte[] header = new byte[DataChecksum.HEADER_LEN];
-    header[0] = (byte) (type.id & 0xff);
-    // Writing in buffer just like DataOutput.WriteInt()
-    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
-    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
-    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
-    header[1+3] = (byte) (bytesPerChecksum & 0xff);
-    return header;
-  }
-  
-  /**
-   * Writes the current checksum to the stream.
-   * If <i>reset</i> is true, then resets the checksum.
-   * @return number of bytes written. Will be equal to getChecksumSize();
-   */
-   public int writeValue( DataOutputStream out, boolean reset )
-                          throws IOException {
-     if ( type.size <= 0 ) {
-       return 0;
-     }
-
-     if ( type.size == 4 ) {
-       out.writeInt( (int) summer.getValue() );
-     } else {
-       throw new IOException( "Unknown Checksum " + type );
-     }
-     
-     if ( reset ) {
-       reset();
-     }
-     
-     return type.size;
-   }
-   
-   /**
-    * Writes the current checksum to a buffer.
-    * If <i>reset</i> is true, then resets the checksum.
-    * @return number of bytes written. Will be equal to getChecksumSize();
-    */
-    public int writeValue( byte[] buf, int offset, boolean reset )
-                           throws IOException {
-      if ( type.size <= 0 ) {
-        return 0;
-      }
-
-      if ( type.size == 4 ) {
-        int checksum = (int) summer.getValue();
-        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
-        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
-        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
-        buf[offset+3] = (byte) (checksum & 0xff);
-      } else {
-        throw new IOException( "Unknown Checksum " + type );
-      }
-      
-      if ( reset ) {
-        reset();
-      }
-      
-      return type.size;
-    }
-   
-   /**
-    * Compares the checksum located at buf[offset] with the current checksum.
-    * @return true if the checksum matches and false otherwise.
-    */
-   public boolean compare( byte buf[], int offset ) {
-     if ( type.size == 4 ) {
-       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
-                      ( (buf[offset+1] & 0xff) << 16 ) |
-                      ( (buf[offset+2] & 0xff) << 8 )  |
-                      ( (buf[offset+3] & 0xff) );
-       return checksum == (int) summer.getValue();
-     }
-     return type.size == 0;
-   }
-   
-  private final Type type;
-  private final Checksum summer;
-  private final int bytesPerChecksum;
-  private int inSum = 0;
-  
-  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
-    this.type = type;
-    summer = checksum;
-    bytesPerChecksum = chunkSize;
-  }
-  
-  // Accessors
-  public Type getChecksumType() {
-    return type;
-  }
-  public int getChecksumSize() {
-    return type.size;
-  }
-  public int getBytesPerChecksum() {
-    return bytesPerChecksum;
-  }
-  public int getNumBytesInSum() {
-    return inSum;
-  }
-  
-  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
-  static public int getChecksumHeaderSize() {
-    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
-  }
-  //Checksum Interface. Just a wrapper around member summer.
-  @Override
-  public long getValue() {
-    return summer.getValue();
-  }
-  @Override
-  public void reset() {
-    summer.reset();
-    inSum = 0;
-  }
-  @Override
-  public void update( byte[] b, int off, int len ) {
-    if ( len > 0 ) {
-      summer.update( b, off, len );
-      inSum += len;
-    }
-  }
-  @Override
-  public void update( int b ) {
-    summer.update( b );
-    inSum += 1;
-  }
-  
-  /**
-   * Verify that the given checksums match the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
-   * but the position is maintained.
-   *  
-   * @param data the DirectByteBuffer pointing to the data to verify.
-   * @param checksums the DirectByteBuffer pointing to a series of stored
-   *                  checksums
-   * @param fileName the name of the file being read, for error-reporting
-   * @param basePos the file position to which the start of 'data' corresponds
-   * @throws ChecksumException if the checksums do not match
-   */
-  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
-      String fileName, long basePos)
-  throws ChecksumException {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      verifyChunkedSums(
-          data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position(),
-          fileName, basePos);
-      return;
-    }
-    if (NativeCrc32.isAvailable()) {
-      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
-          fileName, basePos);
-      return;
-    }
-    
-    int startDataPos = data.position();
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      byte[] sum = new byte[type.size];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        checksums.get(sum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        int calculated = (int)summer.getValue();
-        int stored = (sum[0] << 24 & 0xff000000) |
-          (sum[1] << 16 & 0xff0000) |
-          (sum[2] << 8 & 0xff00) |
-          sum[3] & 0xff;
-        if (calculated != stored) {
-          long errPos = basePos + data.position() - startDataPos - n;
-          throw new ChecksumException(
-              "Checksum error: "+ fileName + " at "+ errPos +
-              " exp: " + stored + " got: " + calculated, errPos);
-        }
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-  
-  /**
-   * Implementation of chunked verification specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void verifyChunkedSums(
-      byte[] data, int dataOff, int dataLen,
-      byte[] checksums, int checksumsOff, String fileName,
-      long basePos) throws ChecksumException {
-    
-    int remaining = dataLen;
-    int dataPos = 0;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      
-      summer.reset();
-      summer.update(data, dataOff + dataPos, n);
-      dataPos += n;
-      remaining -= n;
-      
-      int calculated = (int)summer.getValue();
-      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
-        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
-        (checksums[checksumsOff + 2] << 8 & 0xff00) |
-        checksums[checksumsOff + 3] & 0xff;
-      checksumsOff += 4;
-      if (calculated != stored) {
-        long errPos = basePos + dataPos - n;
-        throw new ChecksumException(
-            "Checksum error: "+ fileName + " at "+ errPos +
-            " exp: " + stored + " got: " + calculated, errPos);
-      }
-    }
-  }
-
-  /**
-   * Calculate checksums for the given data.
-   * 
-   * The 'mark' of the ByteBuffer parameters may be modified by this function,
-   * but the position is maintained.
-   * 
-   * @param data the DirectByteBuffer pointing to the data to checksum.
-   * @param checksums the DirectByteBuffer into which checksums will be
-   *                  stored. Enough space must be available in this
-   *                  buffer to put the checksums.
-   */
-  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
-    if (type.size == 0) return;
-    
-    if (data.hasArray() && checksums.hasArray()) {
-      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
-          checksums.array(), checksums.arrayOffset() + checksums.position());
-      return;
-    }
-    
-    data.mark();
-    checksums.mark();
-    try {
-      byte[] buf = new byte[bytesPerChecksum];
-      while (data.remaining() > 0) {
-        int n = Math.min(data.remaining(), bytesPerChecksum);
-        data.get(buf, 0, n);
-        summer.reset();
-        summer.update(buf, 0, n);
-        checksums.putInt((int)summer.getValue());
-      }
-    } finally {
-      data.reset();
-      checksums.reset();
-    }
-  }
-
-  /**
-   * Implementation of chunked calculation specifically on byte arrays. This
-   * is to avoid the copy when dealing with ByteBuffers that have array backing.
-   */
-  private void calculateChunkedSums(
-      byte[] data, int dataOffset, int dataLength,
-      byte[] sums, int sumsOffset) {
-
-    int remaining = dataLength;
-    while (remaining > 0) {
-      int n = Math.min(remaining, bytesPerChecksum);
-      summer.reset();
-      summer.update(data, dataOffset, n);
-      dataOffset += n;
-      remaining -= n;
-      long calculated = summer.getValue();
-      sums[sumsOffset++] = (byte) (calculated >> 24);
-      sums[sumsOffset++] = (byte) (calculated >> 16);
-      sums[sumsOffset++] = (byte) (calculated >> 8);
-      sums[sumsOffset++] = (byte) (calculated);
-    }
-  }
-
-  @Override
-  public boolean equals(Object other) {
-    if (!(other instanceof DataChecksum)) {
-      return false;
-    }
-    DataChecksum o = (DataChecksum)other;
-    return o.bytesPerChecksum == this.bytesPerChecksum &&
-      o.type == this.type;
-  }
-  
-  @Override
-  public int hashCode() {
-    return (this.type.id + 31) * this.bytesPerChecksum;
-  }
-  
-  @Override
-  public String toString() {
-    return "DataChecksum(type=" + type +
-      ", chunkSize=" + bytesPerChecksum + ")";
-  }
-  
-  /**
-   * This just provides a dummy implimentation for Checksum class
-   * This is used when there is no checksum available or required for 
-   * data
-   */
-  static class ChecksumNull implements Checksum {
-    
-    public ChecksumNull() {}
-    
-    //Dummy interface
-    @Override
-    public long getValue() { return 0; }
-    @Override
-    public void reset() {}
-    @Override
-    public void update(byte[] b, int off, int len) {}
-    @Override
-    public void update(int b) {}
-  };
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumException;
+
+/**
+ * This class provides inteface and utilities for processing checksums for
+ * DFS data transfers.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+@InterfaceStability.Evolving
+public class DataChecksum implements Checksum {
+  
+  // Misc constants
+  public static final int HEADER_LEN = 5; /// 1 byte type and 4 byte len
+  
+  // checksum types
+  public static final int CHECKSUM_NULL    = 0;
+  public static final int CHECKSUM_CRC32   = 1;
+  public static final int CHECKSUM_CRC32C  = 2;
+  public static final int CHECKSUM_DEFAULT = 3; 
+  public static final int CHECKSUM_MIXED   = 4;
+ 
+  /** The checksum types */
+  public static enum Type {
+    NULL  (CHECKSUM_NULL, 0),
+    CRC32 (CHECKSUM_CRC32, 4),
+    CRC32C(CHECKSUM_CRC32C, 4),
+    DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum
+    MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum
+
+    public final int id;
+    public final int size;
+    
+    private Type(int id, int size) {
+      this.id = id;
+      this.size = size;
+    }
+
+    /** @return the type corresponding to the id. */
+    public static Type valueOf(int id) {
+      if (id < 0 || id >= values().length) {
+        throw new IllegalArgumentException("id=" + id
+            + " out of range [0, " + values().length + ")");
+      }
+      return values()[id];
+    }
+  }
+
+
+  public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
+    if ( bytesPerChecksum <= 0 ) {
+      return null;
+    }
+    
+    switch ( type ) {
+    case NULL :
+      return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum );
+    case CRC32 :
+      return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum );
+    case CRC32C:
+      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+    default:
+      return null;  
+    }
+  }
+  
+  /**
+   * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+   * @return DataChecksum of the type in the array or null in case of an error.
+   */
+  public static DataChecksum newDataChecksum( byte bytes[], int offset ) {
+    if ( offset < 0 || bytes.length < offset + HEADER_LEN ) {
+      return null;
+    }
+    
+    // like readInt():
+    int bytesPerChecksum = ( (bytes[offset+1] & 0xff) << 24 ) | 
+                           ( (bytes[offset+2] & 0xff) << 16 ) |
+                           ( (bytes[offset+3] & 0xff) << 8 )  |
+                           ( (bytes[offset+4] & 0xff) );
+    return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum );
+  }
+  
+  /**
+   * This constructucts a DataChecksum by reading HEADER_LEN bytes from
+   * input stream <i>in</i>
+   */
+  public static DataChecksum newDataChecksum( DataInputStream in )
+                                 throws IOException {
+    int type = in.readByte();
+    int bpc = in.readInt();
+    DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc );
+    if ( summer == null ) {
+      throw new IOException( "Could not create DataChecksum of type " +
+                             type + " with bytesPerChecksum " + bpc );
+    }
+    return summer;
+  }
+  
+  /**
+   * Writes the checksum header to the output stream <i>out</i>.
+   */
+  public void writeHeader( DataOutputStream out ) 
+                           throws IOException { 
+    out.writeByte( type.id );
+    out.writeInt( bytesPerChecksum );
+  }
+
+  public byte[] getHeader() {
+    byte[] header = new byte[DataChecksum.HEADER_LEN];
+    header[0] = (byte) (type.id & 0xff);
+    // Writing in buffer just like DataOutput.WriteInt()
+    header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff);
+    header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff);
+    header[1+2] = (byte) ((bytesPerChecksum >>> 8) & 0xff);
+    header[1+3] = (byte) (bytesPerChecksum & 0xff);
+    return header;
+  }
+  
+  /**
+   * Writes the current checksum to the stream.
+   * If <i>reset</i> is true, then resets the checksum.
+   * @return number of bytes written. Will be equal to getChecksumSize();
+   */
+   public int writeValue( DataOutputStream out, boolean reset )
+                          throws IOException {
+     if ( type.size <= 0 ) {
+       return 0;
+     }
+
+     if ( type.size == 4 ) {
+       out.writeInt( (int) summer.getValue() );
+     } else {
+       throw new IOException( "Unknown Checksum " + type );
+     }
+     
+     if ( reset ) {
+       reset();
+     }
+     
+     return type.size;
+   }
+   
+   /**
+    * Writes the current checksum to a buffer.
+    * If <i>reset</i> is true, then resets the checksum.
+    * @return number of bytes written. Will be equal to getChecksumSize();
+    */
+    public int writeValue( byte[] buf, int offset, boolean reset )
+                           throws IOException {
+      if ( type.size <= 0 ) {
+        return 0;
+      }
+
+      if ( type.size == 4 ) {
+        int checksum = (int) summer.getValue();
+        buf[offset+0] = (byte) ((checksum >>> 24) & 0xff);
+        buf[offset+1] = (byte) ((checksum >>> 16) & 0xff);
+        buf[offset+2] = (byte) ((checksum >>> 8) & 0xff);
+        buf[offset+3] = (byte) (checksum & 0xff);
+      } else {
+        throw new IOException( "Unknown Checksum " + type );
+      }
+      
+      if ( reset ) {
+        reset();
+      }
+      
+      return type.size;
+    }
+   
+   /**
+    * Compares the checksum located at buf[offset] with the current checksum.
+    * @return true if the checksum matches and false otherwise.
+    */
+   public boolean compare( byte buf[], int offset ) {
+     if ( type.size == 4 ) {
+       int checksum = ( (buf[offset+0] & 0xff) << 24 ) | 
+                      ( (buf[offset+1] & 0xff) << 16 ) |
+                      ( (buf[offset+2] & 0xff) << 8 )  |
+                      ( (buf[offset+3] & 0xff) );
+       return checksum == (int) summer.getValue();
+     }
+     return type.size == 0;
+   }
+   
+  private final Type type;
+  private final Checksum summer;
+  private final int bytesPerChecksum;
+  private int inSum = 0;
+  
+  private DataChecksum( Type type, Checksum checksum, int chunkSize ) {
+    this.type = type;
+    summer = checksum;
+    bytesPerChecksum = chunkSize;
+  }
+  
+  // Accessors
+  public Type getChecksumType() {
+    return type;
+  }
+  public int getChecksumSize() {
+    return type.size;
+  }
+  public int getBytesPerChecksum() {
+    return bytesPerChecksum;
+  }
+  public int getNumBytesInSum() {
+    return inSum;
+  }
+  
+  public static final int SIZE_OF_INTEGER = Integer.SIZE / Byte.SIZE;
+  static public int getChecksumHeaderSize() {
+    return 1 + SIZE_OF_INTEGER; // type byte, bytesPerChecksum int
+  }
+  //Checksum Interface. Just a wrapper around member summer.
+  @Override
+  public long getValue() {
+    return summer.getValue();
+  }
+  @Override
+  public void reset() {
+    summer.reset();
+    inSum = 0;
+  }
+  @Override
+  public void update( byte[] b, int off, int len ) {
+    if ( len > 0 ) {
+      summer.update( b, off, len );
+      inSum += len;
+    }
+  }
+  @Override
+  public void update( int b ) {
+    summer.update( b );
+    inSum += 1;
+  }
+  
+  /**
+   * Verify that the given checksums match the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,.
+   * but the position is maintained.
+   *  
+   * @param data the DirectByteBuffer pointing to the data to verify.
+   * @param checksums the DirectByteBuffer pointing to a series of stored
+   *                  checksums
+   * @param fileName the name of the file being read, for error-reporting
+   * @param basePos the file position to which the start of 'data' corresponds
+   * @throws ChecksumException if the checksums do not match
+   */
+  public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
+      String fileName, long basePos)
+  throws ChecksumException {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      verifyChunkedSums(
+          data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position(),
+          fileName, basePos);
+      return;
+    }
+    if (NativeCrc32.isAvailable()) {
+      NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
+          fileName, basePos);
+      return;
+    }
+    
+    int startDataPos = data.position();
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      byte[] sum = new byte[type.size];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        checksums.get(sum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        int calculated = (int)summer.getValue();
+        int stored = (sum[0] << 24 & 0xff000000) |
+          (sum[1] << 16 & 0xff0000) |
+          (sum[2] << 8 & 0xff00) |
+          sum[3] & 0xff;
+        if (calculated != stored) {
+          long errPos = basePos + data.position() - startDataPos - n;
+          throw new ChecksumException(
+              "Checksum error: "+ fileName + " at "+ errPos +
+              " exp: " + stored + " got: " + calculated, errPos);
+        }
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+  
+  /**
+   * Implementation of chunked verification specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void verifyChunkedSums(
+      byte[] data, int dataOff, int dataLen,
+      byte[] checksums, int checksumsOff, String fileName,
+      long basePos) throws ChecksumException {
+    
+    int remaining = dataLen;
+    int dataPos = 0;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      
+      summer.reset();
+      summer.update(data, dataOff + dataPos, n);
+      dataPos += n;
+      remaining -= n;
+      
+      int calculated = (int)summer.getValue();
+      int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
+        (checksums[checksumsOff + 1] << 16 & 0xff0000) |
+        (checksums[checksumsOff + 2] << 8 & 0xff00) |
+        checksums[checksumsOff + 3] & 0xff;
+      checksumsOff += 4;
+      if (calculated != stored) {
+        long errPos = basePos + dataPos - n;
+        throw new ChecksumException(
+            "Checksum error: "+ fileName + " at "+ errPos +
+            " exp: " + stored + " got: " + calculated, errPos);
+      }
+    }
+  }
+
+  /**
+   * Calculate checksums for the given data.
+   * 
+   * The 'mark' of the ByteBuffer parameters may be modified by this function,
+   * but the position is maintained.
+   * 
+   * @param data the DirectByteBuffer pointing to the data to checksum.
+   * @param checksums the DirectByteBuffer into which checksums will be
+   *                  stored. Enough space must be available in this
+   *                  buffer to put the checksums.
+   */
+  public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
+    if (type.size == 0) return;
+    
+    if (data.hasArray() && checksums.hasArray()) {
+      calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(),
+          checksums.array(), checksums.arrayOffset() + checksums.position());
+      return;
+    }
+    
+    data.mark();
+    checksums.mark();
+    try {
+      byte[] buf = new byte[bytesPerChecksum];
+      while (data.remaining() > 0) {
+        int n = Math.min(data.remaining(), bytesPerChecksum);
+        data.get(buf, 0, n);
+        summer.reset();
+        summer.update(buf, 0, n);
+        checksums.putInt((int)summer.getValue());
+      }
+    } finally {
+      data.reset();
+      checksums.reset();
+    }
+  }
+
+  /**
+   * Implementation of chunked calculation specifically on byte arrays. This
+   * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   */
+  private void calculateChunkedSums(
+      byte[] data, int dataOffset, int dataLength,
+      byte[] sums, int sumsOffset) {
+
+    int remaining = dataLength;
+    while (remaining > 0) {
+      int n = Math.min(remaining, bytesPerChecksum);
+      summer.reset();
+      summer.update(data, dataOffset, n);
+      dataOffset += n;
+      remaining -= n;
+      long calculated = summer.getValue();
+      sums[sumsOffset++] = (byte) (calculated >> 24);
+      sums[sumsOffset++] = (byte) (calculated >> 16);
+      sums[sumsOffset++] = (byte) (calculated >> 8);
+      sums[sumsOffset++] = (byte) (calculated);
+    }
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (!(other instanceof DataChecksum)) {
+      return false;
+    }
+    DataChecksum o = (DataChecksum)other;
+    return o.bytesPerChecksum == this.bytesPerChecksum &&
+      o.type == this.type;
+  }
+  
+  @Override
+  public int hashCode() {
+    return (this.type.id + 31) * this.bytesPerChecksum;
+  }
+  
+  @Override
+  public String toString() {
+    return "DataChecksum(type=" + type +
+      ", chunkSize=" + bytesPerChecksum + ")";
+  }
+  
+  /**
+   * This just provides a dummy implimentation for Checksum class
+   * This is used when there is no checksum available or required for 
+   * data
+   */
+  static class ChecksumNull implements Checksum {
+    
+    public ChecksumNull() {}
+    
+    //Dummy interface
+    @Override
+    public long getValue() { return 0; }
+    @Override
+    public void reset() {}
+    @Override
+    public void update(byte[] b, int off, int len) {}
+    @Override
+    public void update(int b) {}
+  };
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java

@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Checksum {
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Checksum {
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
 
   /*
    * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
    * See also TestPureJavaCrc32.Table.
    */
-  private static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+  	/* T8_0 */
     0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 
     0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 
     0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 
@@ -154,9 +180,8 @@ public class PureJavaCrc32 implements Checksum {
     0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 
     0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 
     0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 
-    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
-  };
-  private static final int[] T8_1 = new int[] {
+    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+    /* T8_1 */
     0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, 
     0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, 
     0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, 
@@ -220,9 +245,8 @@ public class PureJavaCrc32 implements Checksum {
     0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, 
     0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, 
     0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, 
-    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
-  };
-  private static final int[] T8_2 = new int[] {
+    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+    /* T8_2 */
     0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, 
     0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, 
     0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, 
@@ -286,9 +310,8 @@ public class PureJavaCrc32 implements Checksum {
     0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, 
     0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, 
     0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, 
-    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
-  };
-  private static final int[] T8_3 = new int[] {
+    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+    /* T8_3 */
     0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, 
     0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, 
     0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, 
@@ -352,9 +375,8 @@ public class PureJavaCrc32 implements Checksum {
     0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, 
     0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, 
     0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, 
-    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
-  };
-  private static final int[] T8_4 = new int[] {
+    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+    /* T8_4 */
     0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, 
     0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, 
     0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, 
@@ -418,9 +440,8 @@ public class PureJavaCrc32 implements Checksum {
     0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, 
     0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, 
     0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, 
-    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
-  };
-  private static final int[] T8_5 = new int[] {
+    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+    /* T8_5 */
     0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, 
     0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, 
     0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, 
@@ -484,9 +505,8 @@ public class PureJavaCrc32 implements Checksum {
     0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, 
     0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, 
     0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, 
-    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
-  };
-  private static final int[] T8_6 = new int[] {
+    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+    /* T8_6 */
     0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, 
     0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, 
     0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, 
@@ -550,9 +570,8 @@ public class PureJavaCrc32 implements Checksum {
     0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, 
     0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, 
     0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, 
-    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
-  };
-  private static final int[] T8_7 = new int[] {
+    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+    /* T8_7 */
     0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, 
     0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, 
     0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java

@@ -56,22 +56,38 @@ public class PureJavaCrc32C implements Checksum {
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -80,14 +96,24 @@ public class PureJavaCrc32C implements Checksum {
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
     
   // CRC polynomial tables generated by:
   // java -cp build/test/classes/:build/classes/ \
   //   org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
 
-  static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+    /* T8_0 */
     0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 
     0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 
     0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 
@@ -151,9 +177,8 @@ public class PureJavaCrc32C implements Checksum {
     0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 
     0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 
     0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 
-    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
-  };
-  static final int[] T8_1 = new int[] {
+    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, 
+    /* T8_1 */
     0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 
     0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, 
     0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 
@@ -217,9 +242,8 @@ public class PureJavaCrc32C implements Checksum {
     0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 
     0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 
     0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, 
-    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
-  };
-  static final int[] T8_2 = new int[] {
+    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, 
+    /* T8_2 */
     0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 
     0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, 
     0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 
@@ -283,9 +307,8 @@ public class PureJavaCrc32C implements Checksum {
     0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 
     0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, 
     0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, 
-    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
-  };
-  static final int[] T8_3 = new int[] {
+    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, 
+    /* T8_3 */
     0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 
     0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, 
     0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 
@@ -349,9 +372,8 @@ public class PureJavaCrc32C implements Checksum {
     0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 
     0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, 
     0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, 
-    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
-  };
-  static final int[] T8_4 = new int[] {
+    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, 
+    /* T8_4 */
     0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 
     0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, 
     0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 
@@ -415,9 +437,8 @@ public class PureJavaCrc32C implements Checksum {
     0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 
     0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 
     0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, 
-    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
-  };
-  static final int[] T8_5 = new int[] {
+    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, 
+    /* T8_5 */
     0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 
     0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, 
     0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 
@@ -481,9 +502,8 @@ public class PureJavaCrc32C implements Checksum {
     0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 
     0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 
     0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, 
-    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
-  };
-  static final int[] T8_6 = new int[] {
+    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, 
+    /* T8_6 */
     0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 
     0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, 
     0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 
@@ -547,9 +567,8 @@ public class PureJavaCrc32C implements Checksum {
     0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 
     0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 
     0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, 
-    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
-  };
-  static final int[] T8_7 = new int[] {
+    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, 
+    /* T8_7 */
     0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 
     0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, 
     0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 
@@ -613,6 +632,6 @@ public class PureJavaCrc32C implements Checksum {
     0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 
     0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, 
     0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 
-    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
+    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 
   };
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java

@@ -126,6 +126,10 @@ public class RunJar {
     int firstArg = 0;
     String fileName = args[firstArg++];
     File file = new File(fileName);
+    if (!file.exists() || !file.isFile()) {
+      System.err.println("Not a valid JAR: " + file.getCanonicalPath());
+      System.exit(-1);
+    }
     String mainClassName = null;
 
     JarFile jarFile;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -608,7 +608,8 @@ public class StringUtils {
             "  build = " + VersionInfo.getUrl() + " -r "
                          + VersionInfo.getRevision()  
                          + "; compiled by '" + VersionInfo.getUser()
-                         + "' on " + VersionInfo.getDate()}
+                         + "' on " + VersionInfo.getDate(),
+            "  java = " + System.getProperty("java.version") }
         )
       );
 

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/conffile

@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/etc/hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/control

@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Package: hadoop-common
-Version: @version@
-Section: misc
-Priority: optional
-Provides: hadoop-common
-Architecture: all
-Depends: openjdk-6-jre-headless
-Maintainer: Apache Software Foundation <general@hadoop.apache.org>
-Description: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing.
-Distribution: development

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/postinst

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hadoop-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/postrm

@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-/usr/sbin/groupdel hadoop 2> /dev/null >dev/null
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/preinst

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/prerm

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hadoop-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop \
-  --uninstal

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-datanode

@@ -1,151 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-datanode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-datanode: start and stop the Apache Hadoop Data Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-  DN_USER="root"
-  IDENT_USER=${HADOOP_SECURE_DN_USER}
-else
-  DN_USER="hdfs"
-  IDENT_USER=${DN_USER}
-fi
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-datanode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-datanode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Data Node server not in use (/etc/hadoop/hadoop-datanode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-datanode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-jobtracker

@@ -1,143 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-jobtracker	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Job Tracker server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-jobtracker: start and stop the Apache Hadoop Job Tracker daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-jobtracker_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-jobtracker_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Job Tracker server not in use (/etc/hadoop/hadoop-jobtracker_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid ${JAVA_HOME}/bin/java hadoop-jobtracker && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-jobtracker {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-namenode

@@ -1,155 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-namenode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-namenode: start and stop the Apache Hadoop Name Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-namenode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-namenode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Name Node server not in use (/etc/hadoop/hadoop-namenode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-format() {
-    sudo -u hdfs ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  format)
-	log_daemon_msg "Formatting Apache Hadoop Name Node" "hadoop-namenode"
-	format
-	if [ $? -eq 0 ]; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid ${JAVA_HOME}/bin/java hadoop-namenode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-namenode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-tasktracker

@@ -1,143 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-tasktracker	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Task Tracker server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-tasktracker: start and stop the Apache Hadoop Task Tracker daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-tasktracker_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-tasktracker_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Task Tracker server not in use (/etc/hadoop/hadoop-tasktracker_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid ${JAVA_HOME}/bin/java hadoop-tasktracker && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-tasktracker {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/hadoop-create-user.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-this="$bin/$script"
-
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Require parameter:
-     --config /etc/hadoop                                  Location of Hadoop configuration file
-     -u <username>                                         Create user on HDFS
-  Optional parameters:
-     -h                                                    Display this message
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                 Set Kerberos realm
-     --super-user=hdfs                                     Set super user id
-     --super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'kerberos-realm:' \
-  -l 'super-user:' \
-  -l 'super-user-keytab:' \
-  -o 'h' \
-  -o 'u' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-    exit 1
-fi
-
-create_user() {
-  if [ "${SETUP_USER}" = "" ]; then
-    break
-  fi
-  HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-  export HADOOP_PREFIX
-  export HADOOP_CONF_DIR
-  export JAVA_HOME
-  export SETUP_USER=${SETUP_USER}
-  export SETUP_PATH=/user/${SETUP_USER}
-
-  if [ ! "${KERBEROS_REALM}" = "" ]; then
-    # locate kinit cmd
-    if [ -e /etc/lsb-release ]; then
-      KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    else
-      KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    fi
-    su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER} 
-  fi
-
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 711 ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-
-  if [ "$?" == "0" ]; then
-    echo "User directory has been setup: ${SETUP_PATH}"
-  fi
-}
-
-eval set -- "${OPTS}"
-while true; do
-  case "$1" in
-    -u)
-      shift
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      ;;
-    --super-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      ;;
-    --super-user-keytab)
-      HDFS_USER_KEYTAB=$2; shift 2
-      ;;
-    -h)
-      usage
-      ;; 
-    --)
-      while shift; do
-        SETUP_USER=$1
-        create_user
-      done
-      break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done 
-

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-applications.sh

@@ -1,142 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-this="$bin/$script"
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Require parameter:
-     --config /etc/hadoop                                  Location of Hadoop configuration file
-     --apps=<csl of apps:user hcat:hcat,hbase,hive:user>   Apps you want to setup on hdfs
-                                                           If user is not specified, app name
-                                                           will be used as the user name as well
-  Optional parameters:
-     -h                                                    Display this message
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                 Set Kerberos realm
-     --super-user=hdfs                                     Set super user id
-     --super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'kerberos-realm:' \
-  -l 'super-user:' \
-  -l 'super-user-keytab:' \
-  -l 'apps:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-    exit 1
-fi
-
-function setup_apps
-{
-  if [ -z $APPS ] 
-  then
-    usage
-    break
-  fi
-
-  #if super user is not set default to hdfs
-  HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-
-  if [ ! "${KERBEROS_REALM}" = "" ]; then
-    # locate kinit cmd
-    if [ -e /etc/lsb-release ]; then
-      KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    else
-      KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    fi
-    su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER}
-  fi
-  #process each app
-  oldIFS=$IFS 
-  IFS=','
-  for app in $APPS
-  do
-    IFS=":"
-    arr=($app)
-    app=${arr[0]}
-    user=${arr[1]}
-    IFS=','
-    #if user is empty, default it to app
-    if [ -z $user ]
-    then
-      user=$app
-    fi
-    
-    path="/apps/${app}"
-
-    #create the dir
-    cmd="su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir ${path}' ${HADOOP_HDFS_USER}"
-    echo $cmd
-    eval $cmd
-
-    #make owner to be the app
-    cmd="su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${user} ${path}' ${HADOOP_HDFS_USER}"
-    echo $cmd
-    eval $cmd
-
-    if [ "$?" == "0" ]; then
-      echo "App directory has been setup: ${path}"
-    fi
-  done
-  IFS=$oldIFS
-}
-
-eval set -- "${OPTS}"
-while true; do
-  case "$1" in
-    --apps)
-      APPS=$2; shift 2
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      ;;
-    --super-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      ;;
-    --super-user-keytab)
-      HDFS_USER_KEYTAB=$2; shift 2
-      ;;
-    -h)
-      usage
-      ;; 
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done
-
-setup_apps

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh

@@ -1,707 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-this="${BASH_SOURCE-$0}"
-HADOOP_DEFAULT_PREFIX=`dirname "$this"`/..
-HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}
-export HADOOP_PREFIX
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --auto                                                          Setup path and configuration automatically
-     --default                                                       Setup configuration as default
-     --conf-dir=/etc/hadoop                                          Set configuration directory
-     --datanode-dir=/var/lib/hadoop/hdfs/datanode                    Set datanode directory
-     --group=hadoop                                                  Set Hadoop group name
-     -h                                                              Display this message
-     --hdfs-user=hdfs                                                Set HDFS user
-     --jobtracker-host=hostname                                      Set jobtracker host
-     --namenode-host=hostname                                        Set namenode host
-     --secondarynamenode-host=hostname                               Set secondary namenode host
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-     --kinit-location=/usr/kerberos/bin/kinit                        Set kinit location
-     --keytab-dir=/etc/security/keytabs                              Set keytab directory
-     --log-dir=/var/log/hadoop                                       Set log directory
-     --pid-dir=/var/run/hadoop                                       Set pid directory
-     --hdfs-dir=/var/lib/hadoop/hdfs                                 Set HDFS directory
-     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
-     --mapred-dir=/var/lib/hadoop/mapred                             Set mapreduce directory
-     --mapreduce-user=mr                                             Set mapreduce user
-     --mapreduce-user-keytab=/home/mr/hdfs.keytab                    Set mapreduce user key tab
-     --namenode-dir=/var/lib/hadoop/hdfs/namenode                    Set namenode directory
-     --replication=3                                                 Set replication factor
-     --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler  Set task scheduler
-     --datanodes=hostname1,hostname2,...                             SET the datanodes
-     --tasktrackers=hostname1,hostname2,...                          SET the tasktrackers
-     --dfs-webhdfs-enabled=false|true                                Enable webhdfs
-     --dfs-support-append=false|true                                 Enable append
-     --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts'    Setup proxy users for hadoop
-     --hbase-user=hbase                                              User which hbase is running as. Defaults to hbase
-     --mapreduce-cluster-mapmemory-mb=memory                         Virtual memory of a map slot for the MR framework. Defaults to -1
-     --mapreduce-cluster-reducememory-mb=memory                      Virtual memory, of a reduce slot for the MR framework. Defaults to -1
-     --mapreduce-jobtracker-maxmapmemory-mb=memory                   Maximum virtual memory of a single map task. Defaults to -1
-                                                                     This value should be set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
-     --mapreduce-jobtracker-maxreducememory-mb=memory                Maximum virtual memory of a single reduce task. Defaults to -1
-                                                                     This value should be set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
-     --mapreduce-map-memory-mb=memory                                Virtual memory of a single map slot for a job. Defaults to -1
-                                                                     This value should be <= mapred.cluster.max.map.memory.mb
-     --mapreduce-reduce-memory-mb=memory                             Virtual memory, of a single reduce slot for a job. Defaults to -1
-                                                                     This value should be <= mapred.cluster.max.reduce.memory.mb
-     --dfs-datanode-dir-perm=700                                     Set the permission for the datanode data directories. Defaults to 700
-     --dfs-block-local-path-access-user=user                         User for which you want to enable shortcircuit read.
-     --dfs-client-read-shortcircuit=true/false                       Enable shortcircuit read for the client. Will default to true if the shortcircuit user is set.
-     --dfs-client-read-shortcircuit-skip-checksum=false/true         Disable checking of checksum when shortcircuit read is taking place. Defaults to false.
-  "
-  exit 1
-}
-
-check_permission() {
-  TARGET=$1
-  OWNER="0"
-  RESULT=0
-  while [ "$TARGET" != "/" ]; do
-    if [ "`uname`" = "Darwin" ]; then
-      OWNER=`stat -f %u $TARGET`
-    else
-      OWNER=`stat -c %u $TARGET`
-    fi
-    if [ "$OWNER" != "0" ]; then
-      RESULT=1
-      break
-    fi
-    TARGET=`dirname $TARGET`
-  done
-  return $RESULT
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  if [ -e $2 ]; then
-    mv -f $2 "$2.bak"
-  fi
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-#########################################
-# Function to modify a value of a field in an xml file
-# Params: $1 is the file with full path; $2 is the property, $3 is the new value
-#########################################
-function addPropertyToXMLConf
-{
-  #read the file name with full path
-  local file=$1
-  #get the property name
-  local property=$2
-  #get what value should be set for that
-  local propValue=$3
-  #get the description
-  local desc=$4
-  #get the value for the final tag
-  local finalVal=$5
-
-  #create the property text, make sure the / are escaped
-  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>\n"
-  #if description is not empty add it
-  if [ ! -z $desc ]
-  then
-    propText="${propText}<description>$desc<\/description>\n"
-  fi
-  
-  #if final is not empty add it
-  if [ ! -z $finalVal ]
-  then
-    propText="${propText}final>$finalVal<\/final>\n"
-  fi
-
-  #add the ending tag
-  propText="${propText}<\/property>\n"
-
-  #add the property to the file
-  endText="<\/configuration>"
-  #add the text using sed at the end of the file
-  sed -i "s|$endText|$propText$endText|" $file
-}
-
-##########################################
-# Function to setup up the short circuit read settings
-#########################################
-function setupShortCircuitRead
-{
-  local conf_file="${HADOOP_CONF_DIR}/hdfs-site.xml"
-  #if the shortcircuit user is not set then return
-  if [ -z $DFS_BLOCK_LOCAL_PATH_ACCESS_USER ]
-  then
-    return
-  fi
-  
-  #set the defaults if values not present
-  DFS_CLIENT_READ_SHORTCIRCUIT=${DFS_CLIENT_READ_SHORTCIRCUIT:-false}
-  DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=${DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM:-false}
-
-  #add the user to the conf file
-  addPropertyToXMLConf "$conf_file" "dfs.block.local-path-access.user" "$DFS_BLOCK_LOCAL_PATH_ACCESS_USER"
-  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit" "$DFS_CLIENT_READ_SHORTCIRCUIT"
-  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit.skip.checksum" "$DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM"
-}
-
-##########################################
-# Function to setup up the proxy user settings
-#########################################
-function setupProxyUsers
-{
-  local conf_file="${HADOOP_CONF_DIR}/core-site.xml"
-  #if hadoop proxy users are sent, setup hadoop proxy
-  if [ ! -z $HADOOP_PROXY_USERS ]
-  then
-    oldIFS=$IFS
-    IFS=';'
-    #process each proxy config
-    for proxy in $HADOOP_PROXY_USERS
-    do
-      #get the user, group and hosts information for each proxy
-      IFS=':'
-      arr=($proxy)
-      user="${arr[0]}"
-      groups="${arr[1]}"
-      hosts="${arr[2]}"
-      #determine the property names and values
-      proxy_groups_property="hadoop.proxyuser.${user}.groups"
-      proxy_groups_val="$groups"
-      addPropertyToXMLConf "$conf_file" "$proxy_groups_property" "$proxy_groups_val"
-      proxy_hosts_property="hadoop.proxyuser.${user}.hosts"
-      proxy_hosts_val="$hosts"
-      addPropertyToXMLConf "$conf_file" "$proxy_hosts_property" "$proxy_hosts_val"
-      IFS=';'
-    done
-    IFS=$oldIFS
-  fi
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'auto' \
-  -l 'java-home:' \
-  -l 'conf-dir:' \
-  -l 'default' \
-  -l 'group:' \
-  -l 'hdfs-dir:' \
-  -l 'namenode-dir:' \
-  -l 'datanode-dir:' \
-  -l 'mapred-dir:' \
-  -l 'namenode-host:' \
-  -l 'secondarynamenode-host:' \
-  -l 'jobtracker-host:' \
-  -l 'log-dir:' \
-  -l 'pid-dir:' \
-  -l 'replication:' \
-  -l 'taskscheduler:' \
-  -l 'hdfs-user:' \
-  -l 'hdfs-user-keytab:' \
-  -l 'mapreduce-user:' \
-  -l 'mapreduce-user-keytab:' \
-  -l 'keytab-dir:' \
-  -l 'kerberos-realm:' \
-  -l 'kinit-location:' \
-  -l 'datanodes:' \
-  -l 'tasktrackers:' \
-  -l 'dfs-webhdfs-enabled:' \
-  -l 'hadoop-proxy-users:' \
-  -l 'dfs-support-append:' \
-  -l 'hbase-user:' \
-  -l 'mapreduce-cluster-mapmemory-mb:' \
-  -l 'mapreduce-cluster-reducememory-mb:' \
-  -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
-  -l 'mapreduce-jobtracker-maxreducememory-mb:' \
-  -l 'mapreduce-map-memory-mb:' \
-  -l 'mapreduce-reduce-memory-mb:' \
-  -l 'dfs-datanode-dir-perm:' \
-  -l 'dfs-block-local-path-access-user:' \
-  -l 'dfs-client-read-shortcircuit:' \
-  -l 'dfs-client-read-shortcircuit-skip-checksum:' \
-  -o 'h' \
-  -- "$@") 
-  
-if [ $? != 0 ] ; then
-    usage
-fi
-
-# Make sure the HADOOP_LOG_DIR is not picked up from user environment.
-unset HADOOP_LOG_DIR
-  
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --auto)
-      AUTOSETUP=1
-      AUTOMATED=1
-      shift
-      ;; 
-    --java-home)
-      JAVA_HOME=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --conf-dir)
-      HADOOP_CONF_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --default)
-      AUTOMATED=1; shift
-      ;;
-    --group)
-      HADOOP_GROUP=$2; shift 2
-      AUTOMATED=1
-      ;;
-    -h)
-      usage
-      ;; 
-    --hdfs-dir)
-      HADOOP_HDFS_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --namenode-dir)
-      HADOOP_NN_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --datanode-dir)
-      HADOOP_DN_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --mapred-dir)
-      HADOOP_MAPRED_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --namenode-host)
-      HADOOP_NN_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --secondarynamenode-host)
-      HADOOP_SNN_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --jobtracker-host)
-      HADOOP_JT_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --log-dir)
-      HADOOP_LOG_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --pid-dir)
-      HADOOP_PID_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --replication)
-      HADOOP_REPLICATION=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --taskscheduler)
-      HADOOP_TASK_SCHEDULER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user)
-      HADOOP_MR_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --keytab-dir)
-      KEYTAB_DIR=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user-keytab)
-      HDFS_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user-keytab)
-      MR_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      SECURITY_TYPE="kerberos"
-      AUTOMATED=1
-      ;;
-    --kinit-location)
-      KINIT=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --datanodes)
-      DATANODES=$2; shift 2
-      AUTOMATED=1
-      DATANODES=$(echo $DATANODES | tr ',' ' ')
-      ;;
-    --tasktrackers)
-      TASKTRACKERS=$2; shift 2
-      AUTOMATED=1
-      TASKTRACKERS=$(echo $TASKTRACKERS | tr ',' ' ')
-      ;;
-    --dfs-webhdfs-enabled)
-      DFS_WEBHDFS_ENABLED=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hadoop-proxy-users)
-      HADOOP_PROXY_USERS=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-support-append)
-      DFS_SUPPORT_APPEND=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hbase-user)
-      HBASE_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-cluster-mapmemory-mb)
-      MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-cluster-reducememory-mb)
-      MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-jobtracker-maxmapmemory-mb)
-      MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-jobtracker-maxreducememory-mb)
-      MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-map-memory-mb)
-      MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-reduce-memory-mb)
-      MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-datanode-dir-perm)
-      DFS_DATANODE_DIR_PERM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-block-local-path-access-user)
-      DFS_BLOCK_LOCAL_PATH_ACCESS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-client-read-shortcircuit)
-      DFS_CLIENT_READ_SHORTCIRCUIT=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-client-read-shortcircuit-skip-checksum)
-      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done 
-
-AUTOSETUP=${AUTOSETUP:-1}
-JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
-HADOOP_NN_HOST=${HADOOP_NN_HOST:-`hostname`}
-HADOOP_SNN_HOST=${HADOOP_SNN_HOST:-`hostname`}
-HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
-HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`}
-HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs}
-HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
-HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop}
-HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/log/hadoop}
-HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
-HADOOP_REPLICATION=${HADOOP_RELICATION:-3}
-HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
-HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
-DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
-DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
-HBASE_USER=${HBASE_USER:-hbase}
-MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
-MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
-MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
-MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
-MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
-MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
-KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
-HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
-MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
-DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
-DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
-KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
-SECURITY_TYPE=${SECURITY_TYPE:-simple}
-KINIT=${KINIT:-/usr/kerberos/bin/kinit}
-#deault the data dir perm to 700
-DFS_DATANODE_DIR_PERM=${DFS_DATANODE_DIR_PERM:-700}
-if [ "${SECURITY_TYPE}" = "kerberos" ]; then
-  TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
-  HADOOP_DN_ADDR="0.0.0.0:1019"
-  HADOOP_DN_HTTP_ADDR="0.0.0.0:1022"
-  SECURITY="true"
-  HADOOP_SECURE_DN_USER=${HADOOP_HDFS_USER}
-else
-  TASK_CONTROLLER="org.apache.hadoop.mapred.DefaultTaskController"
-  HADOOP_DN_ADDR="0.0.0.0:50010"
-  HADOOP_DN_HTTP_ADDR="0.0.0.0:50075"
-  SECURITY="false"
-  HADOOP_SECURE_DN_USER=""
-fi
-
-#unset env vars
-unset HADOOP_CLIENT_OPTS HADOOP_NAMENODE_OPTS HADOOP_DATANODE_OPTS HADOOP_SECONDARYNAMENODE_OPTS HADOOP_JAVA_PLATFORM_OPTS
-
-if [ "${AUTOMATED}" != "1" ]; then
-  echo "Setup Hadoop Configuration"
-  echo
-  echo -n "Where would you like to put config directory? (${HADOOP_CONF_DIR}) "
-  read USER_HADOOP_CONF_DIR
-  echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) "
-  read USER_HADOOP_LOG_DIR
-  echo -n "Where would you like to put pid directory? (${HADOOP_PID_DIR}) "
-  read USER_HADOOP_PID_DIR
-  echo -n "What is the host of the namenode? (${HADOOP_NN_HOST}) "
-  read USER_HADOOP_NN_HOST
-  echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) "
-  read USER_HADOOP_NN_DIR
-  echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) "
-  read USER_HADOOP_DN_DIR
-  echo -n "What is the host of the jobtracker? (${HADOOP_JT_HOST}) "
-  read USER_HADOOP_JT_HOST
-  echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) "
-  read USER_HADOOP_MAPRED_DIR
-  echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) "
-  read USER_JAVA_HOME
-  echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) "
-  read USER_AUTOSETUP
-  echo
-  JAVA_HOME=${USER_USER_JAVA_HOME:-$JAVA_HOME}
-  HADOOP_NN_HOST=${USER_HADOOP_NN_HOST:-$HADOOP_NN_HOST}
-  HADOOP_NN_DIR=${USER_HADOOP_NN_DIR:-$HADOOP_NN_DIR}
-  HADOOP_DN_DIR=${USER_HADOOP_DN_DIR:-$HADOOP_DN_DIR}
-  HADOOP_JT_HOST=${USER_HADOOP_JT_HOST:-$HADOOP_JT_HOST}
-  HADOOP_HDFS_DIR=${USER_HADOOP_HDFS_DIR:-$HADOOP_HDFS_DIR}
-  HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR}
-  HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
-  HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR}
-  HADOOP_PID_DIR=${USER_HADOOP_PID_DIR:-$HADOOP_PID_DIR}
-  HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR}
-  AUTOSETUP=${USER_AUTOSETUP:-y}
-  echo "Review your choices:"
-  echo
-  echo "Config directory            : ${HADOOP_CONF_DIR}"
-  echo "Log directory               : ${HADOOP_LOG_DIR}"
-  echo "PID directory               : ${HADOOP_PID_DIR}"
-  echo "Namenode host               : ${HADOOP_NN_HOST}"
-  echo "Namenode directory          : ${HADOOP_NN_DIR}"
-  echo "Datanode directory          : ${HADOOP_DN_DIR}"
-  echo "Jobtracker host             : ${HADOOP_JT_HOST}"
-  echo "Mapreduce directory         : ${HADOOP_MAPRED_DIR}"
-  echo "Task scheduler              : ${HADOOP_TASK_SCHEDULER}"
-  echo "JAVA_HOME directory         : ${JAVA_HOME}"
-  echo "Create dirs/copy conf files : ${AUTOSETUP}"
-  echo
-  echo -n "Proceed with generate configuration? (y/N) "
-  read CONFIRM
-  if [ "${CONFIRM}" != "y" ]; then
-    echo "User aborted setup, exiting..."
-    exit 1
-  fi
-fi
-
-if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
-  if [ -d ${KEYTAB_DIR} ]; then
-    chmod 700 ${KEYTAB_DIR}/*
-    chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[jt]t.service.keytab
-    chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[dns]n.service.keytab
-  fi
-  chmod 755 -R ${HADOOP_PREFIX}/sbin/*hadoop*
-  chmod 755 -R ${HADOOP_PREFIX}/bin/hadoop
-
-  HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-${HADOOP_PREFIX}/libexec}
-  chmod 755 -R ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
-
-  mkdir -p /home/${HADOOP_MR_USER}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} /home/${HADOOP_MR_USER}
-  HDFS_DIR=`echo ${HADOOP_HDFS_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${HDFS_DIR}
-  if [ -e ${HADOOP_NN_DIR} ]; then
-    rm -rf ${HADOOP_NN_DIR}
-  fi
-  DATANODE_DIR=`echo ${HADOOP_DN_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${DATANODE_DIR}
-  MAPRED_DIR=`echo ${HADOOP_MAPRED_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${MAPRED_DIR}
-  mkdir -p ${HADOOP_CONF_DIR}
-  check_permission ${HADOOP_CONF_DIR}
-  if [ $? == 1 ]; then
-    echo "Full path to ${HADOOP_CONF_DIR} should be owned by root."
-    exit 1
-  fi
-
-  mkdir -p ${HADOOP_LOG_DIR}
-  #create the log sub dir for diff users
-  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-
-  mkdir -p ${HADOOP_PID_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HDFS_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${DATANODE_DIR}
-  chmod 700 -R ${DATANODE_DIR}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${MAPRED_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_PID_DIR}
-  chown root:${HADOOP_GROUP} ${HADOOP_PID_DIR}
-
-  #change the permission and the owner
-  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-
-  #setup up the proxy users
-  setupProxyUsers
- 
-  #setup short circuit read
-  setupShortCircuitRead
-
-  #set the owner of the hadoop dir to root
-  chown root ${HADOOP_PREFIX}
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
-  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
-  
-  #set taskcontroller
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
-  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
-
-
-  #generate the slaves file and include and exclude files for hdfs and mapred
-  echo '' > ${HADOOP_CONF_DIR}/slaves
-  echo '' > ${HADOOP_CONF_DIR}/dfs.include
-  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
-  echo '' > ${HADOOP_CONF_DIR}/mapred.include
-  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
-  for dn in $DATANODES
-  do
-    echo $dn >> ${HADOOP_CONF_DIR}/slaves
-    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
-  done
-  for tt in $TASKTRACKERS
-  do
-    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
-  done
-
-  echo "Configuration setup is completed."
-  if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then
-    echo "Proceed to run hadoop-setup-hdfs.sh on namenode."
-  fi
-else
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-  
-  #setup up the proxy users
-  setupProxyUsers
-  
-  #setup short circuit read
-  setupShortCircuitRead
-
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
-  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
-  #set taskcontroller
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
-  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
-  
-  #generate the slaves file and include and exclude files for hdfs and mapred
-  echo '' > ${HADOOP_CONF_DIR}/slaves
-  echo '' > ${HADOOP_CONF_DIR}/dfs.include
-  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
-  echo '' > ${HADOOP_CONF_DIR}/mapred.include
-  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
-  for dn in $DATANODES
-  do
-    echo $dn >> ${HADOOP_CONF_DIR}/slaves
-    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
-  done
-  for tt in $TASKTRACKERS
-  do
-    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
-  done
-  
-  echo
-  echo "Configuration file has been generated in:"
-  echo
-  echo "${HADOOP_CONF_DIR}/core-site.xml"
-  echo "${HADOOP_CONF_DIR}/hdfs-site.xml"
-  echo "${HADOOP_CONF_DIR}/mapred-site.xml"
-  echo "${HADOOP_CONF_DIR}/hadoop-env.sh"
-  echo "${HADOOP_CONF_DIR}/hadoop-policy.xml"
-  echo "${HADOOP_CONF_DIR}/commons-logging.properties"
-  echo "${HADOOP_CONF_DIR}/taskcontroller.cfg"
-  echo "${HADOOP_CONF_DIR}/capacity-scheduler.xml"
-  echo "${HADOOP_CONF_DIR}/log4j.properties"
-  echo "${HADOOP_CONF_DIR}/hadoop-metrics2.properties"
-  echo
-  echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode."
-fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-hdfs.sh

@@ -1,157 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --format                                                        Force namenode format
-     --group=hadoop                                                  Set Hadoop group
-     -h                                                              Display this message
-     --hdfs-user=hdfs                                                Set HDFS user
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
-     --mapreduce-user=mr                                             Set mapreduce user
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'format' \
-  -l 'hdfs-user:' \
-  -l 'hdfs-user-keytab:' \
-  -l 'mapreduce-user:' \
-  -l 'kerberos-realm:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --format)
-      FORMAT_NAMENODE=1; shift
-      AUTOMATED=1
-      ;;
-    --group)
-      HADOOP_GROUP=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user)
-      HADOOP_MR_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --yarn-user)
-      HADOOP_YARN_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user-keytab)
-      HDFS_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
-HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-HADOOP_MAPREDUCE_USER=${HADOOP_MR_USER:-mapred}
-
-if [ "${KERBEROS_REALM}" != "" ]; then
-  # Determine kerberos location base on Linux distro.
-  if [ -e /etc/lsb-release ]; then
-    KERBEROS_BIN=/usr/bin
-  else
-    KERBEROS_BIN=/usr/kerberos/bin
-  fi
-  kinit_cmd="${KERBEROS_BIN}/kinit -k -t ${HDFS_KEYTAB} ${HADOOP_HDFS_USER}"
-  su -c "${kinit_cmd}" ${HADOOP_HDFS_USER}
-fi
-
-echo "Setup Hadoop Distributed File System"
-echo
-
-# Format namenode
-if [ "${FORMAT_NAMENODE}" == "1" ]; then
-  echo "Formatting namenode"
-  echo
-  su -c "echo Y | ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format" ${HADOOP_HDFS_USER}
-  echo
-fi
-
-# Start namenode process
-echo "Starting namenode process"
-echo
-if [ -e ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh ]; then
-  DAEMON_PATH=${HADOOP_PREFIX}/sbin
-else
-  DAEMON_PATH=${HADOOP_PREFIX}/bin
-fi
-su -c "${DAEMON_PATH}/hadoop-daemon.sh --config ${HADOOP_CONF_DIR} start namenode" ${HADOOP_HDFS_USER}
-echo
-echo "Initialize HDFS file system: "
-echo
-
-#create the /user dir 
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user" ${HADOOP_HDFS_USER}
-
-#create /tmp and give it 777
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp" ${HADOOP_HDFS_USER}
-
-#create /mapred
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /mapred" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 700 /mapred" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${HADOOP_MAPREDUCE_USER}:system /mapred" ${HADOOP_HDFS_USER}
-
-if [ $? -eq 0 ]; then
-  echo "Completed."
-else
-  echo "Unknown error occurred, check hadoop logs for details."
-fi
-
-echo
-echo "Please startup datanode processes: /etc/init.d/hadoop-datanode start"

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-single-node.sh

@@ -1,219 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script for setup HDFS file system for single node deployment
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --default                   Setup system as default
-     -h                          Display this message
-  "
-  exit 1
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'default' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-if [ -e /etc/hadoop/hadoop-env.sh ]; then
-  . /etc/hadoop/hadoop-env.sh
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --default)
-      AUTOMATED=1; shift
-      ;;
-    -h)
-      usage
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-if [ "${AUTOMATED}" != "1" ]; then
-  echo "Welcome to Hadoop single node setup wizard"
-  echo
-  echo -n "Would you like to use default single node configuration? (y/n) "
-  read SET_CONFIG
-  echo -n "Would you like to format name node? (y/n) "
-  read SET_FORMAT
-  echo -n "Would you like to setup default directory structure? (y/n) "
-  read SET_MKDIR
-  echo -n "Would you like to start up Hadoop? (y/n) "
-  read STARTUP
-  echo -n "Would you like to start up Hadoop on reboot? (y/n) "
-  read SET_REBOOT
-  echo
-  echo "Review your choices:"
-  echo
-  echo "Setup single node configuration    : ${SET_CONFIG}"
-  echo "Format namenode                    : ${SET_FORMAT}"
-  echo "Setup default file system structure: ${SET_MKDIR}"
-  echo "Start up Hadoop                    : ${STARTUP}"
-  echo "Start up Hadoop on reboot          : ${SET_REBOOT}"
-  echo
-  echo -n "Proceed with setup? (y/n) "
-  read CONFIRM
-  if [ "${CONFIRM}" != "y" ]; then
-    echo "User aborted setup, exiting..."
-    exit 1
-  fi
-else
-  SET_CONFIG="y"
-  SET_FORMAT="y"
-  SET_MKDIR="y"
-  STARTUP="y"
-  SET_REBOOT="y"
-fi
-
-AUTOMATED=${AUTOMATED:-0}
-SET_CONFIG=${SET_CONFIG:-y}
-SET_FORMAT=${SET_FORMAT:-n}
-SET_MKDIR=${SET_MKDIR:-y}
-STARTUP=${STARTUP:-y}
-SET_REBOOT=${SET_REBOOT:-y}
-
-# Make sure system is not already started
-/etc/init.d/hadoop-namenode stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-datanode stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null
-
-if [ "${SET_CONFIG}" == "y" ]; then
-  JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-  HADOOP_NN_HOST=${HADOOP_NN_HOST:-localhost}
-  HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
-  HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-  HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost}
-  HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs}
-  HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
-  HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
-  HADOOP_LOG_DIR="/var/log/hadoop"
-  HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
-  HADOOP_REPLICATION=${HADOOP_RELICATION:-1}
-  ${HADOOP_PREFIX}/sbin/hadoop-setup-conf.sh --auto \
-    --hdfs-user=hdfs \
-    --mapreduce-user=mapred \
-    --conf-dir=${HADOOP_CONF_DIR} \
-    --datanode-dir=${HADOOP_DN_DIR} \
-    --hdfs-dir=${HADOOP_HDFS_DIR} \
-    --jobtracker-host=${HADOOP_JT_HOST} \
-    --log-dir=${HADOOP_LOG_DIR} \
-    --pid-dir=${HADOOP_PID_DIR} \
-    --mapred-dir=${HADOOP_MAPRED_DIR} \
-    --namenode-dir=${HADOOP_NN_DIR} \
-    --namenode-host=${HADOOP_NN_HOST} \
-    --replication=${HADOOP_REPLICATION}
-fi
-
-if [ ! -e ${HADOOP_NN_DIR} ]; then
-  rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
-  mkdir -p ${HADOOP_HDFS_DIR}
-  chmod 755 ${HADOOP_HDFS_DIR}
-  chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  /etc/init.d/hadoop-namenode format
-elif [ "${SET_FORMAT}" == "y" ]; then
-  rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
-  mkdir -p ${HADOOP_HDFS_DIR}
-  chmod 755 ${HADOOP_HDFS_DIR}
-  chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  rm -rf ${HADOOP_NN_DIR}
-  /etc/init.d/hadoop-namenode format
-fi
-
-/etc/init.d/hadoop-namenode start
-/etc/init.d/hadoop-datanode start
-
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown mapred:mapred /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp' hdfs
-
-/etc/init.d/hadoop-jobtracker start
-/etc/init.d/hadoop-tasktracker start
-
-if [ "${SET_REBOOT}" == "y" ]; then
-  if [ -e /etc/debian_version ]; then
-    ln -sf ../init.d/hadoop-namenode /etc/rc2.d/S90hadoop-namenode
-    ln -sf ../init.d/hadoop-datanode /etc/rc2.d/S91hadoop-datanode
-    ln -sf ../init.d/hadoop-jobtracker /etc/rc2.d/S92hadoop-jobtracker
-    ln -sf ../init.d/hadoop-tasktracker /etc/rc2.d/S93hadoop-tasktracker
-    ln -sf ../init.d/hadoop-namenode /etc/rc6.d/S10hadoop-namenode
-    ln -sf ../init.d/hadoop-datanode /etc/rc6.d/S11hadoop-datanode
-    ln -sf ../init.d/hadoop-jobtracker /etc/rc6.d/S12hadoop-jobtracker
-    ln -sf ../init.d/hadoop-tasktracker /etc/rc6.d/S13hadoop-tasktracker
-  elif [ -e /etc/redhat-release ]; then
-    /sbin/chkconfig hadoop-namenode --add
-    /sbin/chkconfig hadoop-datanode --add
-    /sbin/chkconfig hadoop-jobtracker --add
-    /sbin/chkconfig hadoop-tasktracker --add
-    /sbin/chkconfig hadoop-namenode on
-    /sbin/chkconfig hadoop-datanode on
-    /sbin/chkconfig hadoop-jobtracker on
-    /sbin/chkconfig hadoop-tasktracker on
-  fi
-fi
-
-if [ "${STARTUP}" != "y" ]; then
-  /etc/init.d/hadoop-namenode stop
-  /etc/init.d/hadoop-datanode stop
-  /etc/init.d/hadoop-jobtracker stop
-  /etc/init.d/hadoop-tasktracker stop
-fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-validate-setup.sh

@@ -1,183 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-
-###############################################################################
-# Run the following jobs to validate a hadoop cluster
-## teragen
-## terasort
-## teravalidate
-# If they all pass 0 will be returned and 1 otherwise
-# The test will work for both secure and unsecure deploys. If the kerberos-realm
-# is passed we will assume that the deploy is secure and proceed with a kinit before
-# running the validation jobs.
-################################################################################
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     -h                                                              Display this message
-     --user=hdfs
-     --user_keytab=/home/hdfs/hdfs.keytab
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'user:' \
-  -l 'user-keytab:' \
-  -l 'kerberos-realm:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --user)
-      TEST_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --user-keytab)
-      USER_KEYTAB_FILE=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-#set the hadoop command and the path to the hadoop examples jar
-HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
-
-#find the hadoop examples jar
-HADOOP_EXAMPLES_JAR=''
-
-#find under HADOOP_PREFIX (tar ball install)
-HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-mapreduce-examples-*.jar' | head -n1`
-
-#if its not found look under /usr/share/hadoop (rpm/deb installs)
-if [ "$HADOOP_EXAMPLES_JAR" == '' ]
-then
-  HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-mapreduce-examples-*.jar' | head -n1`
-fi
-
-#if it is still empty then dont run the tests
-if [ "$HADOOP_EXAMPLES_JAR" == '' ]
-then
-  echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
-  exit 1
-fi
-
-# do a kinit if secure
-if [ "${KERBEROS_REALM}" != "" ]; then
-  # Determine kerberos location base on Linux distro.
-  if [ -e /etc/lsb-release ]; then
-    KERBEROS_BIN=/usr/bin
-  else
-    KERBEROS_BIN=/usr/kerberos/bin
-  fi
-  kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
-  echo $kinit_cmd
-  eval $kinit_cmd
-  if [ $? -ne 0 ]
-  then
-    echo "kinit command did not run successfully."
-    exit 1
-  fi
-fi
-
-
-#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
-PARENT_DIR="validate_deploy_`date +%s`"
-TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
-TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
-TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
-#tera gen cmd
-TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
-
-#tera sort cmd
-TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
-
-#tera validate cmd
-TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
-
-echo "Starting teragen...."
-
-#run tera gen
-echo $TERA_GEN_CMD
-eval $TERA_GEN_CMD
-if [ $? -ne 0 ]; then
-  echo "tera gen failed."
-  exit 1
-fi
-
-echo "Teragen passed starting terasort...."
-
-
-#run tera sort
-echo $TERA_SORT_CMD
-eval $TERA_SORT_CMD
-if [ $? -ne 0 ]; then
-  echo "tera sort failed."
-  exit 1
-fi
-
-echo "Terasort passed starting teravalidate...."
-
-#run tera validate
-echo $TERA_VALIDATE_CMD
-eval $TERA_VALIDATE_CMD
-if [ $? -ne 0 ]; then
-  echo "tera validate failed."
-  exit 1
-fi
-
-echo "teragen, terasort, teravalidate passed."
-echo "Cleaning the data created by tests: $PARENT_DIR"
-
-CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
-echo $CLEANUP_CMD
-eval $CLEANUP_CMD
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-datanode

@@ -1,93 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop datanode
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop datanode
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid"
-desc="Hadoop datanode daemon"
-HADOOP_PREFIX="/usr"
-
-start() {
-  echo -n $"Starting $desc (hadoop-datanode): "
-  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
-  else
-    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
-  fi
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-datanode
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-datanode): "
-  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
-  else
-    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
-  fi
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-datanode $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-datanode ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-jobtracker

@@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop jobtracker
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop jobtracker
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid"
-desc="Hadoop jobtracker daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-jobtracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start jobtracker
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-jobtracker
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-jobtracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop jobtracker
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-jobtracker $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-jobtracker ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-namenode

@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop namenode
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop namenode
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid"
-desc="Hadoop namenode daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-namenode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start namenode $1
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-namenode
-  return $RETVAL
-}
-
-upgrade() {
-  start -upgrade
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-namenode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop namenode
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-namenode $PIDFILE
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-restart() {
-  stop
-  start
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-namenode ] && restart || :
-}
-
-format() {
-  daemon --user hdfs ${HADOOP_PREFIX}/bin/hadoop namenode -format
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  upgrade)
-    upgrade
-    ;;
-  format)
-    format
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart|try-restart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|try-restart|upgrade}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-tasktracker

@@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop tasktracker
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop tasktracker
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid"
-desc="Hadoop tasktracker daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-tasktracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start tasktracker
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-tasktracker
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-tasktracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop tasktracker
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-tasktracker $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-tasktracker ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/spec/hadoop.spec

@@ -1,174 +0,0 @@
-#   Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  See the NOTICE file distributed with
-#   this work for additional information regarding copyright ownership.
-#   The ASF licenses this file to You under the Apache License, Version 2.0
-#   (the "License"); you may not use this file except in compliance with
-#   the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#
-# RPM Spec file for Hadoop version @version@
-#
-
-%define name         hadoop-common
-%define version      @version@
-%define release      @package.release@
-
-# Installation Locations
-%define _prefix      @package.prefix@
-%define _bin_dir     %{_prefix}/bin
-%define _conf_dir    @package.conf.dir@
-%define _lib_dir     %{_prefix}/lib
-%define _lib64_dir   %{_prefix}/lib64
-%define _libexec_dir %{_prefix}/libexec
-%define _log_dir     @package.log.dir@
-%define _pid_dir     @package.pid.dir@
-%define _sbin_dir    %{_prefix}/sbin
-%define _share_dir   %{_prefix}/share
-%define _var_dir     @package.var.dir@
-
-# Build time settings
-%define _build_dir  @package.build.dir@
-%define _final_name @final.name@
-%define debug_package %{nil}
-
-# Disable brp-java-repack-jars for aspect J
-%define __os_install_post    \
-    /usr/lib/rpm/redhat/brp-compress \
-    %{!?__debug_package:/usr/lib/rpm/redhat/brp-strip %{__strip}} \
-    /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
-    /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \
-    /usr/lib/rpm/brp-python-bytecompile %{nil}
-
-# RPM searches perl files for dependancies and this breaks for non packaged perl lib
-# like thrift so disable this
-%define _use_internal_dependency_generator 0
-
-%ifarch i386
-%global hadoop_arch Linux-i386-32
-%endif
-%ifarch amd64 x86_64
-%global hadoop_arch Linux-amd64-64
-%endif
-%ifarch noarch
-%global hadoop_arch ""
-%endif
-
-Summary: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing
-License: Apache License, Version 2.0
-URL: http://hadoop.apache.org/core/
-Vendor: Apache Software Foundation
-Group: Development/Libraries
-Name: %{name}
-Version: %{version}
-Release: %{release} 
-Source0: %{_final_name}-bin.tar.gz
-Prefix: %{_prefix}
-Prefix: %{_conf_dir}
-Prefix: %{_log_dir}
-Prefix: %{_pid_dir}
-Buildroot: %{_build_dir}
-Requires: sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, jdk >= 1.6
-AutoReqProv: no
-Provides: hadoop
-
-%description
-The Apache Hadoop project develops open-source software for reliable, scalable, 
-distributed computing.  Hadoop includes these subprojects:
-
-Hadoop Common: The common utilities that support the other Hadoop subprojects.
-
-%prep
-%setup -n %{_final_name}
-
-%build
-if [ -d ${RPM_BUILD_DIR}%{_prefix} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_prefix}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_log_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_log_dir}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_conf_dir}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_pid_dir}
-fi
-
-mkdir -p ${RPM_BUILD_DIR}%{_prefix}
-mkdir -p ${RPM_BUILD_DIR}%{_bin_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_lib_dir}
-%ifarch amd64 x86_64
-mkdir -p ${RPM_BUILD_DIR}%{_lib64_dir}
-%endif
-mkdir -p ${RPM_BUILD_DIR}%{_libexec_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_log_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_conf_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_pid_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_sbin_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_share_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_var_dir}
-
-#########################
-#### INSTALL SECTION ####
-#########################
-%install
-mv ${RPM_BUILD_DIR}/%{_final_name}/bin/* ${RPM_BUILD_DIR}%{_bin_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/etc/hadoop/* ${RPM_BUILD_DIR}%{_conf_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/lib/* ${RPM_BUILD_DIR}%{_lib_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/libexec/* ${RPM_BUILD_DIR}%{_libexec_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/sbin/* ${RPM_BUILD_DIR}%{_sbin_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/share/* ${RPM_BUILD_DIR}%{_share_dir}
-rm -rf ${RPM_BUILD_DIR}/%{_final_name}/etc
-
-%pre
-getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop
-
-%post
-bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \
-       --prefix=${RPM_INSTALL_PREFIX0} \
-       --bin-dir=${RPM_INSTALL_PREFIX0}/bin \
-       --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \
-       --conf-dir=${RPM_INSTALL_PREFIX1} \
-       --log-dir=${RPM_INSTALL_PREFIX2} \
-       --pid-dir=${RPM_INSTALL_PREFIX3}
-
-%preun
-bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \
-       --prefix=${RPM_INSTALL_PREFIX0} \
-       --bin-dir=${RPM_INSTALL_PREFIX0}/bin \
-       --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \
-       --conf-dir=${RPM_INSTALL_PREFIX1} \
-       --log-dir=${RPM_INSTALL_PREFIX2} \
-       --pid-dir=${RPM_INSTALL_PREFIX3} \
-       --uninstall
-
-%files 
-%defattr(-,root,root)
-%attr(0755,root,hadoop) %{_log_dir}
-%attr(0775,root,hadoop) %{_pid_dir}
-%config(noreplace) %{_conf_dir}/configuration.xsl
-%config(noreplace) %{_conf_dir}/core-site.xml
-%config(noreplace) %{_conf_dir}/hadoop-env.sh
-%config(noreplace) %{_conf_dir}/hadoop-metrics.properties
-%config(noreplace) %{_conf_dir}/hadoop-metrics2.properties
-%config(noreplace) %{_conf_dir}/hadoop-policy.xml
-%config(noreplace) %{_conf_dir}/log4j.properties
-%config(noreplace) %{_conf_dir}/masters
-%config(noreplace) %{_conf_dir}/slaves
-%config{noreplace) %{_conf_dir}/fair-scheduler.xml
-%{_conf_dir}/hadoop-env.sh.template
-%{_conf_dir}/ssl-client.xml.example
-%{_conf_dir}/ssl-server.xml.example
-%{_prefix}
-

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/capacity-scheduler.xml

@@ -1,198 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<!-- This is the configuration file for the resource manager in Hadoop. -->
-<!-- You can configure various scheduling parameters related to queues. -->
-<!-- The properties for a queue follow a naming convention,such as, -->
-<!-- mapred.capacity-scheduler.queue.<queue-name>.property-name. -->
-
-<configuration>
-
-  <property>
-    <name>mapred.capacity-scheduler.maximum-system-jobs</name>
-    <value>3000</value>
-    <description>Maximum number of jobs in the system which can be initialized,
-     concurrently, by the CapacityScheduler.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.capacity</name>
-    <value>100</value>
-    <description>Percentage of the number of slots in the cluster that are
-      to be available for jobs in this queue.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-capacity</name>
-    <value>-1</value>
-    <description>
-	maximum-capacity defines a limit beyond which a queue cannot use the capacity of the cluster.
-	This provides a means to limit how much excess capacity a queue can use. By default, there is no limit.
-	The maximum-capacity of a queue can only be greater than or equal to its minimum capacity.
-        Default value of -1 implies a queue can use complete capacity of the cluster.
-
-        This property could be to curtail certain jobs which are long running in nature from occupying more than a 
-        certain percentage of the cluster, which in the absence of pre-emption, could lead to capacity guarantees of 
-        other queues being affected.
-        
-        One important thing to note is that maximum-capacity is a percentage , so based on the cluster's capacity
-        the max capacity would change. So if large no of nodes or racks get added to the cluster , max Capacity in 
-        absolute terms would increase accordingly.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.minimum-user-limit-percent</name>
-    <value>100</value>
-    <description> Each queue enforces a limit on the percentage of resources 
-    allocated to a user at any given time, if there is competition for them. 
-    This user limit can vary between a minimum and maximum value. The former
-    depends on the number of users who have submitted jobs, and the latter is
-    set to this property value. For example, suppose the value of this 
-    property is 25. If two users have submitted jobs to a queue, no single 
-    user can use more than 50% of the queue resources. If a third user submits
-    a job, no single user can use more than 33% of the queue resources. With 4 
-    or more users, no user can use more than 25% of the queue's resources. A 
-    value of 100 implies no user limits are imposed. 
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.user-limit-factor</name>
-    <value>1</value>
-    <description>The multiple of the queue capacity which can be configured to 
-    allow a single user to acquire more slots. 
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks</name>
-    <value>200000</value>
-    <description>The maximum number of tasks, across all jobs in the queue, 
-    which can be initialized concurrently. Once the queue's jobs exceed this 
-    limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user</name>
-    <value>100000</value>
-    <description>The maximum number of tasks per-user, across all the of the 
-    user's jobs in the queue, which can be initialized concurrently. Once the 
-    user's jobs exceed this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.init-accept-jobs-factor</name>
-    <value>10</value>
-    <description>The multipe of (maximum-system-jobs * queue-capacity) used to 
-    determine the number of jobs which are accepted by the scheduler.  
-    </description>
-  </property>
-
-  <!-- The default configuration settings for the capacity task scheduler -->
-  <!-- The default values would be applied to all the queues which don't have -->
-  <!-- the appropriate property for the particular queue -->
-  <property>
-    <name>mapred.capacity-scheduler.default-supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions by default in a job queue.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
-    <value>100</value>
-    <description>The percentage of the resources limited to a particular user
-      for the job queue at any given point of time by default.
-    </description>
-  </property>
-
-
-  <property>
-    <name>mapred.capacity-scheduler.default-user-limit-factor</name>
-    <value>1</value>
-    <description>The default multiple of queue-capacity which is used to 
-    determine the amount of slots a single user can consume concurrently.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-queue</name>
-    <value>200000</value>
-    <description>The default maximum number of tasks, across all jobs in the 
-    queue, which can be initialized concurrently. Once the queue's jobs exceed 
-    this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-user</name>
-    <value>100000</value>
-    <description>The default maximum number of tasks per-user, across all the of 
-    the user's jobs in the queue, which can be initialized concurrently. Once 
-    the user's jobs exceed this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-init-accept-jobs-factor</name>
-    <value>10</value>
-    <description>The default multipe of (maximum-system-jobs * queue-capacity) 
-    used to determine the number of jobs which are accepted by the scheduler.  
-    </description>
-  </property>
-
-  <!-- Capacity scheduler Job Initialization configuration parameters -->
-  <property>
-    <name>mapred.capacity-scheduler.init-poll-interval</name>
-    <value>5000</value>
-    <description>The amount of time in miliseconds which is used to poll 
-    the job queues for jobs to initialize.
-    </description>
-  </property>
-  <property>
-    <name>mapred.capacity-scheduler.init-worker-threads</name>
-    <value>5</value>
-    <description>Number of worker threads which would be used by
-    Initialization poller to initialize jobs in a set of queue.
-    If number mentioned in property is equal to number of job queues
-    then a single thread would initialize jobs in a queue. If lesser
-    then a thread would get a set of queues assigned. If the number
-    is greater then number of threads would be equal to number of 
-    job queues.
-    </description>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/commons-logging.properties

@@ -1,25 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#Logging Implementation
-
-#Log4J
-org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
-
-#JDK Logger
-#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/core-site.xml

@@ -1,100 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
-  <property>
-    <name>local.realm</name>
-    <value>${KERBEROS_REALM}</value>
-  </property>
-
-  <!-- file system properties -->
-
-  <property>
-    <name>fs.default.name</name>
-    <value>hdfs://${HADOOP_NN_HOST}:8020</value>
-    <description>The name of the default file system.  Either the
-      literal string "local" or a host:port for NDFS.
-    </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>fs.trash.interval</name>
-    <value>360</value>
-    <description>Number of minutes between trash checkpoints.
-      If zero, the trash feature is disabled.
-    </description>
-  </property>
-  
-  <property>
-    <name>hadoop.security.auth_to_local</name>
-    <value>
-	RULE:[2:$1@$0]([jt]t@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0]([nd]n@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](mapred@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0](hdfs@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](mapredqa@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0](hdfsqa@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](hm@.*${KERBEROS_REALM})s/.*/${HBASE_USER}/
-        RULE:[2:$1@$0](rs@.*${KERBEROS_REALM})s/.*/${HBASE_USER}/
-        DEFAULT
-    </value>
-    <description></description>
-  </property>
-
-  <property>
-    <name>hadoop.security.authentication</name>
-    <value>${SECURITY_TYPE}</value>
-    <description>
-      Set the authentication for the cluster. Valid values are: simple or
-      kerberos.
-    </description>
-  </property>
-
-  <property>
-    <name>hadoop.security.authorization</name>
-    <value>${SECURITY}</value>
-    <description>
-      Enable authorization for different protocols.
-    </description>
-  </property>
-
-  <property>
-    <name>hadoop.security.groups.cache.secs</name>
-    <value>14400</value>
-  </property>
-
-  <property>
-    <name>hadoop.kerberos.kinit.command</name>
-    <value>${KINIT}</value>
-  </property>
-
-  <property>
-    <name>hadoop.http.filter.initializers</name>
-    <value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh

@@ -1,78 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Set Hadoop-specific environment variables here.
-
-# The only required environment variable is JAVA_HOME.  All others are
-# optional.  When running a distributed configuration it is best to
-# set JAVA_HOME in this file, so that it is correctly defined on
-# remote nodes.
-
-# The java implementation to use.
-export JAVA_HOME=${JAVA_HOME}
-
-# The jsvc implementation to use. Jsvc is required to run secure datanodes.
-#export JSVC_HOME=${JSVC_HOME}
-
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
-
-# Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
-for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
-  if [ "$HADOOP_CLASSPATH" ]; then
-    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
-  else
-    export HADOOP_CLASSPATH=$f
-  fi
-done
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-#export HADOOP_HEAPSIZE=
-#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
-
-# Extra Java runtime options.  Empty by default.
-export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS"
-
-# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
-export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
-
-export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
-
-# The ZKFC does not need a large heap, and keeping it small avoids
-# any potential for long GC pauses
-export HADOOP_ZKFC_OPTS="-Xmx256m $HADOOP_ZKFC_OPTS"
-
-# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS"
-#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
-
-# On secure datanodes, user to run the datanode as after dropping privileges
-export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
-
-# Where log files are stored.  $HADOOP_HOME/logs by default.
-#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
-
-# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-
-# The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=${HADOOP_PID_DIR}
-export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
-
-# A string representing this instance of hadoop. $USER by default.
-export HADOOP_IDENT_STRING=$USER

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-metrics2.properties

@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# syntax: [prefix].[source|sink|jmx].[instance].[options]
-# See package.html for org.apache.hadoop.metrics2 for details
-
-*.period=60
-

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml

@@ -1,250 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-  <property>
-    <name>security.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientProtocol, which is used by user code
-    via the DistributedFileSystem.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.client.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
-    for block recovery.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for DatanodeProtocol, which is used by datanodes to
-    communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.inter.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
-    for updating generation timestamp.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.namenode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for NamenodeProtocol, the protocol used by the secondary
-    namenode to communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.inter.tracker.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterTrackerProtocol, used by the tasktrackers to
-    communicate with the jobtracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
- 
-  <property>
-    <name>security.job.submission.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for JobSubmissionProtocol, used by job clients to
-    communciate with the jobtracker for job submission, querying job status etc.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.task.umbilical.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
-    tasks to communicate with the parent tasktracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
- <property>
-    <name>security.admin.operations.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for AdminOperationsProtocol. Used for admin commands.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.usertogroups.mappings.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for RefreshUserMappingsProtocol. Used to refresh
-    users mappings. The ACL is a comma-separated list of user and
-    group names. The user and group list is separated by a blank. For
-    e.g. "alice,bob users,wheel".  A special value of "*" means all
-    users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.policy.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
-    dfsadmin and mradmin commands to refresh the security policy in-effect.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-
-  <!-- YARN Protocols -->
-
-  <property>
-    <name>security.resourcetracker.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
-    <description>ACL for ResourceTracker protocol, used by the
-    ResourceManager and NodeManager to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.admin.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
-    <description>ACL for RMAdminProtocol, for admin commands. 
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.client.resourcemanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientRMProtocol, used by the ResourceManager 
-    and applications submission clients to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationmaster.resourcemanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for AMRMProtocol, used by the ResourceManager 
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.containermanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ContainerManager protocol, used by the NodeManager 
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.resourcelocalizer.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ResourceLocalizer protocol, used by the NodeManager 
-    and ResourceLocalizer to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.task.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
-    tasks to communicate with the parent tasktracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for MRClientProtocol, used by job clients to
-    communciate with the MR ApplicationMaster to query job status etc. 
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-  
-  <property>
-    <name>security.ha.service.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for HAService protocol used by HAAdmin to manage the
-      active and stand-by states of namenode.</description>
-  </property>
-  <property>
-    <name>security.zkfc.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for access to the ZK Failover Controller
-    </description>
-  </property>
-
-   <property>
-      <name>security.mrhs.client.protocol.acl</name>
-      <value>*</value>
-      <description>ACL for HSClientProtocol, used by job clients to
-       communciate with the MR History Server job status etc. 
-       The ACL is a comma-separated list of user and group names. The user and
-       group list is separated by a blank. For e.g. "alice,bob users,wheel".
-       A special value of "*" means all users are allowed.</description>
-    </property>
-  
-  <property>
-    <name>security.qjournal.service.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for QJournalProtocol, used by the NN to communicate with
-    JNs when using the QuorumJournalManager for edit logs.</description>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml

@@ -1,269 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<configuration>
-
-<!-- file system properties -->
-
-  <property>
-    <name>dfs.namenode.name.dir</name>
-    <value>${HADOOP_NN_DIR}</value>
-    <description>Determines where on the local filesystem the DFS name node
-      should store the name table.  If this is a comma-delimited list
-      of directories then the name table is replicated in all of the
-      directories, for redundancy. </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>dfs.datanode.data.dir</name>
-    <value>${HADOOP_DN_DIR}</value>
-    <description>Determines where on the local filesystem an DFS data node
-       should store its blocks.  If this is a comma-delimited
-       list of directories, then data will be stored in all named
-       directories, typically on different devices.
-       Directories that do not exist are ignored.
-    </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>dfs.namenode.safemode.threshold-pct</name>
-    <value>1.0f</value>
-    <description>
-        Specifies the percentage of blocks that should satisfy the minimal 
-        replication requirement defined by dfs.namenode.replication.min. 
-        Values less than or equal to 0 mean not to start in safe mode.
-        Values greater than 1 will make safe mode permanent.
-        </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.address</name>
-    <value>${HADOOP_DN_ADDR}</value>
-  </property>
-
-  <property>
-    <name>dfs.datanode.http.address</name>
-    <value>${HADOOP_DN_HTTP_ADDR}</value>
-  </property>
-
-  <property>
-    <name>dfs.namenode.http-address</name>
-    <value>${HADOOP_NN_HOST}:50070</value>
-    <description>The name of the default file system.  Either the
-       literal string "local" or a host:port for NDFS.
-    </description>
-    <final>true</final>
-  </property>
-
-  <!-- Permissions configuration -->
-  <property>
-    <name>dfs.umaskmode</name>
-    <value>077</value>
-    <description>
-      The octal umask used when creating files and directories.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.block.access.token.enable</name>
-    <value>${SECURITY}</value>
-    <description>
-      Are access tokens are used as capabilities for accessing datanodes.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.kerberos.principal</name>
-    <value>nn/_HOST@${local.realm}</value>
-    <description>
-      Kerberos principal name for the NameNode
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.kerberos.principal</name>
-    <value>nn/_HOST@${local.realm}</value>
-    <description>
-        Kerberos principal name for the secondary NameNode.
-    </description>
-  </property>
-
-
-  <property>
-    <name>dfs.namenode.kerberos.https.principal</name>
-    <value>host/_HOST@${local.realm}</value>
-    <description>
-       The Kerberos principal for the host that the NameNode runs on.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.kerberos.https.principal</name>
-    <value>host/_HOST@${local.realm}</value>
-    <description>
-      The Kerberos principal for the hostthat the secondary NameNode runs on.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.kerberos.principal</name>
-    <value>dn/_HOST@${local.realm}</value>
-    <description>
-      The Kerberos principal that the DataNode runs as. "_HOST" is replaced by 
-      the real host name.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.web.authentication.kerberos.principal</name>
-    <value>HTTP/_HOST@${local.realm}</value>
-    <description>
-      The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
-
-      The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos
-      HTTP SPNEGO specification.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.web.authentication.kerberos.keytab</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      The Kerberos keytab file with the credentials for the
-      HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.keytab.file</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      Combined keytab file containing the namenode service and host principals.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.keytab.file</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      Combined keytab file containing the namenode service and host principals.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.keytab.file</name>
-    <value>/etc/security/keytabs/dn.service.keytab</value>
-    <description>
-        The filename of the keytab file for the DataNode.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.https.port</name>
-    <value>50470</value>
-    <description>The https port where namenode binds</description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.https-address</name>
-    <value>${HADOOP_NN_HOST}:50470</value>
-    <description>The https address where namenode binds</description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.data.dir.perm</name>
-    <value>${DFS_DATANODE_DIR_PERM}</value>
-    <description>The permissions that should be there on dfs.datanode.data.dir
-      directories. The datanode will not come up if the permissions are
-      different on existing dfs.datanode.data.dir directories. If the directories
-      don't exist, they will be created with this permission.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.cluster.administrators</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for who all can view the default servlets in the HDFS</description>
-  </property>
-  
-  <property>
-    <name>dfs.permissions.superusergroup</name>
-    <value>${HADOOP_GROUP}</value>
-    <description>The name of the group of super-users.</description>
-  </property>
-  
-  <property>
-    <name>dfs.namenode.http-address</name>
-    <value>${HADOOP_NN_HOST}:50070</value>
-    <description>
-      The address and the base port where the dfs namenode web ui will listen on.
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.https-address</name>
-    <value>${HADOOP_NN_HOST}:50470</value>
-  </property>
-
-  <property>
-    <name>dfs.namenode.secondary.http-address</name>
-    <value>${HADOOP_SNN_HOST}:50090</value>
-    <description>
-      The secondary namenode http server address and port.
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.hosts</name>
-    <value>${HADOOP_CONF_DIR}/dfs.include</value>
-    <description>Names a file that contains a list of hosts that are
-      permitted to connect to the namenode. The full pathname of the file
-      must be specified.  If the value is empty, all hosts are
-      permitted.</description>
-  </property>
-
-  <property>
-    <name>dfs.hosts.exclude</name>
-    <value>${HADOOP_CONF_DIR}/dfs.exclude</value>
-    <description>Names a file that contains a list of hosts that are
-      not permitted to connect to the namenode.  The full pathname of the
-      file must be specified.  If the value is empty, no hosts are
-      excluded.
-    </description>
-  </property>
-  <property>
-    <name>dfs.webhdfs.enabled</name>
-    <value>${DFS_WEBHDFS_ENABLED}</value>
-    <description>Enable or disable webhdfs. Defaults to false</description>
-  </property>
-  <property>
-    <name>dfs.support.append</name>
-    <value>${DFS_SUPPORT_APPEND}</value>
-    <description>Enable or disable append. Defaults to false</description>
-  </property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties

@@ -1,212 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=ALL
-
-# Null Appender
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Rolling File Appender - cap space usage at 5gb.
-#
-hadoop.log.maxfilesize=256MB
-hadoop.log.maxbackupindex=20
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
-log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this 
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-#Security appender
-#
-hadoop.security.logger=INFO,NullAppender
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth-${user.name}.audit
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# Daily Rolling Security appender
-#
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,NullAppender
-hdfs.audit.log.maxfilesize=256MB
-hdfs.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
-log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,NullAppender
-mapred.audit.log.maxfilesize=256MB
-mapred.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
-log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-#
-# Job Summary Appender 
-#
-# Use following logger to send summary to separate file defined by 
-# hadoop.mapreduce.jobsummary.log.file :
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-# 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
-hadoop.mapreduce.jobsummary.log.maxbackupindex=20
-log4j.appender.JSA=org.apache.log4j.RollingFileAppender
-log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
-log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
-log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
-#
-# Yarn ResourceManager Application Summary Log 
-#
-# Set the ResourceManager summary log filename
-#yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
-# Set the ResourceManager summary log level and appender
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-#    - hadoop.log.dir (Hadoop Log directory)
-#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-
-#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
-#log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-#log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
-#log4j.appender.RMSUMMARY.MaxFileSize=256MB
-#log4j.appender.RMSUMMARY.MaxBackupIndex=20
-#log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-#log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-queue-acls.xml

@@ -1,33 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<configuration>
-<property>
-<name>mapred.queue.default.acl-submit-job</name>
-<value>*</value>
-</property>
-<property>
-<name>mapred.queue.default.acl-administer-jobs</name>
-<value>*</value>
-</property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml

@@ -1,308 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- 
-  <property>
-    <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
-    <value>250</value>
-    <description>Normally, this is the amount of time before killing
-      processes, and the recommended-default is 5.000 seconds - a value of
-      5000 here.  In this case, we are using it solely to blast tasks before
-      killing them, and killing them very quickly (1/4 second) to guarantee
-      that we do not leave VMs around for later jobs.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.system.dir</name>
-    <value>/mapred/mapredsystem</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker</name>
-    <value>${HADOOP_JT_HOST}:9000</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.http.address</name>
-    <value>${HADOOP_JT_HOST}:50030</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.local.dir</name>
-    <value>${HADOOP_MAPRED_DIR}</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.administrators</name>
-    <value>${HADOOP_MR_USER}</value>
-  </property>
-
-  <property>
-    <name>mapred.map.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some map tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.reduce.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some reduce tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.output.compression.type</name>
-    <value>BLOCK</value>
-    <description>If the job outputs are to compressed as SequenceFiles, how 
-       should they be compressed? Should be one of NONE, RECORD or BLOCK.
-    </description>
-  </property>
-
-  <property>
-    <name>jetty.connector</name>
-    <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
-  </property>
-
-  <property>
-    <name>mapred.task.tracker.task-controller</name>
-    <value>${TASK_CONTROLLER}</value>
-  </property>
-
-  <property>
-    <name>mapred.child.root.logger</name>
-    <value>INFO,TLA</value>
-  </property>
-
-  <property>
-    <name>mapred.child.java.opts</name>
-    <value>-server -Xmx640m -Djava.net.preferIPv4Stack=true</value>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.persist.jobstatus.active</name>
-    <value>true</value>
-    <description>Indicates if persistency of job status information is
-      active or not.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.persist.jobstatus.dir</name>
-    <value>file:///${HADOOP_LOG_DIR}/${HADOOP_MR_USER}/jobstatus</value>
-    <description>The directory where the job status information is persisted
-      in a file system to be available after it drops of the memory queue and
-      between jobtracker restarts.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.history.completed.location</name>
-    <value>/mapred/history/done</value>
-  </property>
-
-  <property>
-    <name>mapred.heartbeats.in.second</name>
-    <value>200</value>
-    <description>to enable HADOOP:5784</description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.outofband.heartbeat</name>
-    <value>true</value>
-    <description>to enable MAPREDUCE:270</description>
-  </property>
-
-  <property>
-    <name>mapred.jobtracker.maxtasks.per.job</name>
-    <value>200000</value>
-    <final>true</final>
-    <description>The maximum number of tasks for a single job.
-      A value of -1 indicates that there is no maximum.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.kerberos.principal</name>
-    <value>jt/_HOST@${local.realm}</value>   
-    <description>
-       JT principal
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.kerberos.principal</name>
-    <value>tt/_HOST@${local.realm}</value>   
-    <description>       
-       TT principal.
-    </description>
-  </property>
-
-
-  <property>
-    <name>hadoop.job.history.user.location</name>
-    <value>none</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.keytab.file</name>
-    <value>/etc/security/keytabs/jt.service.keytab</value>
-    <description>
-        The keytab for the jobtracker principal.
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.keytab.file</name>
-    <value>/etc/security/keytabs/tt.service.keytab</value>
-    <description>The filename of the keytab for the task tracker</description>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.staging.root.dir</name>
-    <value>/user</value>
-    <description>The Path prefix for where the staging directories should be 
-      placed. The next level is always the user's
-      name. It is a path in the default file system.
-    </description>
-  </property>
-
-
-  <property>
-    <name>mapreduce.job.acl-modify-job</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapreduce.job.acl-view-job</name>
-    <value>Dr.Who</value>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.group</name>
-    <value>${HADOOP_GROUP}</value>
-    <description>The group that the task controller uses for accessing the
-      task controller. The mapred user must be a member and users should *not*
-      be members.
-    </description> 
-  </property>
-
-  <property>
-    <name>mapred.acls.enabled</name>
-    <value>true</value>
-  </property>
-  
-  <property>
-    <name>mapred.jobtracker.taskScheduler</name>
-    <value>org.apache.hadoop.mapred.CapacityTaskScheduler</value>
-  </property>
-  <property>
-    <name>mapred.queue.names</name>
-    <value>default</value>
-  </property>
-
-  <!-- settings for the history server -->
-  <property>
-    <name>mapreduce.history.server.embedded</name>
-    <value>false</value>
-  </property>
-  <property>
-    <name>mapreduce.history.server.http.address</name>
-    <value>${HADOOP_JT_HOST}:51111</value>
-  </property>
-  <property>
-    <name>mapreduce.jobhistory.kerberos.principal</name>
-    <value>jt/_HOST@${local.realm}</value>   
-    <description>history server principal</description>
-  </property>
-  <property>
-    <name>mapreduce.jobhistory.keytab.file</name>
-    <value>/etc/security/keytabs/jt.service.keytab</value>
-    <description>
-        The keytab for the jobtracker principal.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.hosts</name>
-    <value>${HADOOP_CONF_DIR}/mapred.include</value>
-    <description>Names a file that contains the list of nodes that may
-    connect to the jobtracker.  If the value is empty, all hosts are
-    permitted.</description>
-  </property>
-          
-  <property>
-    <name>mapred.hosts.exclude</name>
-    <value>${HADOOP_CONF_DIR}/mapred.exclude</value>
-    <description>Names a file that contains the list of hosts that
-      should be excluded by the jobtracker.  If the value is empty, no
-      hosts are excluded.</description>
-  </property>
-  <property>
-    <name>mapred.jobtracker.retirejob.check</name>
-    <value>10000</value>
-  </property>
-  <property>
-    <name>mapred.jobtracker.retirejob.interval</name>
-    <value>0</value>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.mapmemory.mb</name>
-    <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.reducememory.mb</name>
-    <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.maxmapmemory.mb</name>
-    <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.maxreducememory.mb</name>
-    <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.map.memory.mb</name>
-    <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.reduce.memory.mb</name>
-    <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
-  </property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/taskcontroller.cfg

@@ -1,21 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-mapreduce.cluster.local.dir=${HADOOP_MAPRED_DIR}
-mapreduce.tasktracker.group=${HADOOP_GROUP}
-hadoop.log.dir=${HADOOP_LOG_DIR}/${HADOOP_MR_USER}

hadoop-common-project/hadoop-common/src/main/packages/update-hadoop-env.sh

@@ -1,172 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script configures hadoop-env.sh and symlinkis directories for 
-# relocating RPM locations.
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Required parameters:
-     --prefix=PREFIX             path to install into
-
-  Optional parameters:
-     --arch=i386                 OS Architecture
-     --bin-dir=PREFIX/bin        Executable directory
-     --conf-dir=/etc/hadoop      Configuration directory
-     --log-dir=/var/log/hadoop   Log directory
-     --pid-dir=/var/run          PID file location
-     --sbin-dir=PREFIX/sbin      System executable directory
-  "
-  exit 1
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'arch:' \
-  -l 'prefix:' \
-  -l 'bin-dir:' \
-  -l 'conf-dir:' \
-  -l 'lib-dir:' \
-  -l 'log-dir:' \
-  -l 'pid-dir:' \
-  -l 'sbin-dir:' \
-  -l 'uninstall' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --arch)
-      ARCH=$2 ; shift 2
-      ;;
-    --prefix)
-      PREFIX=$2 ; shift 2
-      ;;
-    --bin-dir)
-      BIN_DIR=$2 ; shift 2
-      ;;
-    --log-dir)
-      LOG_DIR=$2 ; shift 2
-      ;;
-    --lib-dir)
-      LIB_DIR=$2 ; shift 2
-      ;;
-    --conf-dir)
-      CONF_DIR=$2 ; shift 2
-      ;;
-    --pid-dir)
-      PID_DIR=$2 ; shift 2
-      ;;
-    --sbin-dir)
-      SBIN_DIR=$2 ; shift 2
-      ;;
-    --uninstall)
-      UNINSTALL=1; shift
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-for var in PREFIX; do
-  if [ -z "$(eval "echo \$$var")" ]; then
-    echo Missing param: $var
-    usage
-  fi
-done
-
-ARCH=${ARCH:-i386}
-HADOOP_PREFIX=$PREFIX
-HADOOP_BIN_DIR=${BIN_DIR:-$PREFIX/bin}
-HADOOP_CONF_DIR=${CONF_DIR:-$PREFIX/etc/hadoop}
-HADOOP_LIB_DIR=${LIB_DIR:-$PREFIX/lib}
-HADOOP_LOG_DIR=${LOG_DIR:-$PREFIX/var/log}
-HADOOP_PID_DIR=${PID_DIR:-$PREFIX/var/run}
-HADOOP_SBIN_DIR=${SBIN_DIR:-$PREFIX/sbin}
-UNINSTALL=${UNINSTALL:-0}
-
-if [ "${ARCH}" != "i386" ]; then
-  HADOOP_LIB_DIR=${HADOOP_LIB_DIR}64
-fi
-
-if [ "${UNINSTALL}" -eq "1" ]; then
-  # Remove symlinks
-  if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
-    rm -rf ${HADOOP_PREFIX}/etc/hadoop
-  fi
-  rm -f /etc/default/hadoop-env.sh
-  if [ -d /etc/profile.d ]; then
-    rm -f /etc/profile.d/hadoop-env.sh
-  fi
-else
-  # Create symlinks
-  if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
-    mkdir -p ${HADOOP_PREFIX}/etc
-    ln -sf ${HADOOP_CONF_DIR} ${HADOOP_PREFIX}/etc/hadoop
-  fi
-  ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/default/hadoop-env.sh
-  if [ -d /etc/profile.d ]; then
-    ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh
-  fi
-
-  mkdir -p ${HADOOP_LOG_DIR}
-  chown root:hadoop ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_LOG_DIR}
-
-  if [ ! -d ${HADOOP_PID_DIR} ]; then
-    mkdir -p ${HADOOP_PID_DIR}
-    chown root:hadoop ${HADOOP_PID_DIR}
-    chmod 775 ${HADOOP_PID_DIR}
-  fi
-
-  TFILE="/tmp/$(basename $0).$$.tmp"
-  if [ -z "${JAVA_HOME}" ]; then
-    if [ -e /etc/debian_version ]; then
-      JAVA_HOME=`update-alternatives --config java | grep java | cut -f2 -d':' | cut -f2 -d' ' | sed -e 's/\/bin\/java//'`
-    else
-      JAVA_HOME=/usr/java/default
-    fi
-  fi
-  template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template $TFILE
-  cp ${TFILE} ${CONF_DIR}/hadoop-env.sh
-  rm -f ${TFILE}
-fi

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java

@@ -472,7 +472,9 @@ public class TestFileUtil {
     setupDirs();
 
     long du = FileUtil.getDU(TEST_DIR);
-    //Only two files (in partitioned) have 4 bytes each
-    Assert.assertEquals(du, 8);
+    // Only two files (in partitioned).  Each has 3 characters + system-specific
+    // line separator.
+    long expected = 2 * (3 + System.getProperty("line.separator").length());
+    Assert.assertEquals(expected, du);
   }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java

@@ -730,6 +730,55 @@ public class TestCodec {
     }
   }
 
+  @Test
+  public void testGzipLongOverflow() throws IOException {
+    LOG.info("testGzipLongOverflow");
+
+    // Don't use native libs for this test.
+    Configuration conf = new Configuration();
+    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
+    assertFalse("ZlibFactory is using native libs against request",
+        ZlibFactory.isNativeZlibLoaded(conf));
+
+    // Ensure that the CodecPool has a BuiltInZlibInflater in it.
+    Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+    assertNotNull("zlibDecompressor is null!", zlibDecompressor);
+    assertTrue("ZlibFactory returned unexpected inflator",
+        zlibDecompressor instanceof BuiltInZlibInflater);
+    CodecPool.returnDecompressor(zlibDecompressor);
+
+    // Now create a GZip text file.
+    String tmpDir = System.getProperty("test.build.data", "/tmp/");
+    Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
+    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+      new GZIPOutputStream(new FileOutputStream(f.toString()))));
+
+    final int NBUF = 1024 * 4 + 1;
+    final char[] buf = new char[1024 * 1024];
+    for (int i = 0; i < buf.length; i++) buf[i] = '\0';
+    for (int i = 0; i < NBUF; i++) {
+      bw.write(buf);
+    }
+    bw.close();
+
+    // Now read it back, using the CodecPool to establish the
+    // decompressor to use.
+    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
+    CompressionCodec codec = ccf.getCodec(f);
+    Decompressor decompressor = CodecPool.getDecompressor(codec);
+    FileSystem fs = FileSystem.getLocal(conf);
+    InputStream is = fs.open(f);
+    is = codec.createInputStream(is, decompressor);
+    BufferedReader br = new BufferedReader(new InputStreamReader(is));
+    for (int j = 0; j < NBUF; j++) {
+      int n = br.read(buf);
+      assertEquals("got wrong read length!", n, buf.length);
+      for (int i = 0; i < buf.length; i++)
+        assertEquals("got wrong byte!", buf[i], '\0');
+    }
+    br.close();
+  }
+
   public void testGzipCodecWrite(boolean useNative) throws IOException {
     // Create a gzipped file using a compressor from the CodecPool,
     // and try to read it back via the regular GZIPInputStream.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java

@@ -141,7 +141,7 @@ public class TestVLong extends TestCase {
       int shift = rng.nextInt(Long.SIZE) + 1;
       long mask = (1L << shift) - 1;
       long a = ((long) rng.nextInt()) << 32;
-      long b = ((long) rng.nextInt()) & 0xffffffff;
+      long b = ((long) rng.nextInt()) & 0xffffffffL;
       data[i] = (a + b) & mask;
     }
     

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java

@@ -19,10 +19,7 @@
 package org.apache.hadoop.ipc;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
+import static org.junit.Assert.*;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
@@ -44,6 +41,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.SaslInputStream;
 import org.apache.hadoop.security.SaslRpcClient;
@@ -449,23 +447,100 @@ public class TestSaslRPC {
     }
     System.out.println("Test is successful.");
   }
-  
+
+  // insecure -> insecure
+  @Test
+  public void testInsecureClientInsecureServer() throws Exception {
+    assertEquals(AuthenticationMethod.SIMPLE,
+                 getAuthMethod(false, false, false));
+  }
+
+  @Test
+  public void testInsecureClientInsecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(false, false, true));
+  }
+
+  // insecure -> secure
+  @Test
+  public void testInsecureClientSecureServer() throws Exception {
+    RemoteException e = null;
+    try {
+      getAuthMethod(false, true, false);
+    } catch (RemoteException re) {
+      e = re;
+    }
+    assertNotNull(e);
+    assertEquals(AccessControlException.class.getName(), e.getClassName());
+  }
+
+  @Test
+  public void testInsecureClientSecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(false, true, true));
+  }
+
+  // secure -> secure
+  @Test
+  public void testSecureClientSecureServer() throws Exception {
+    /* Should be this when multiple secure auths are supported and we can
+     * dummy one out:
+     *     assertEquals(AuthenticationMethod.SECURE_AUTH_METHOD,
+     *                  getAuthMethod(true, true, false));
+     */
+    try {
+      getAuthMethod(true, true, false);
+    } catch (IOException ioe) {
+      // can't actually test kerberos w/o kerberos...
+      String expectedError = "Failed to specify server's Kerberos principal";
+      String actualError = ioe.getMessage();
+      assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+          actualError.contains(expectedError));
+    }
+  }
+
   @Test
-  public void testDigestAuthMethodSecureServer() throws Exception {
-    checkDigestAuthMethod(true);
+  public void testSecureClientSecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(true, true, true));
   }
 
+  // secure -> insecure
   @Test
-  public void testDigestAuthMethodInsecureServer() throws Exception {
-    checkDigestAuthMethod(false);
+  public void testSecureClientInsecureServerWithToken() throws Exception {
+    assertEquals(AuthenticationMethod.TOKEN,
+                 getAuthMethod(true, false, true));
   }
 
-  private void checkDigestAuthMethod(boolean secureServer) throws Exception {
+  @Test
+  public void testSecureClientInsecureServer() throws Exception {
+    /* Should be this when multiple secure auths are supported and we can
+     * dummy one out:
+     *     assertEquals(AuthenticationMethod.SIMPLE
+     *                  getAuthMethod(true, false, false));
+     */
+    try {
+      getAuthMethod(true, false, false);
+    } catch (IOException ioe) {
+      // can't actually test kerberos w/o kerberos...
+      String expectedError = "Failed to specify server's Kerberos principal";
+      String actualError = ioe.getMessage();
+      assertTrue("["+actualError+"] doesn't start with ["+expectedError+"]",
+          actualError.contains(expectedError));
+    }
+  }
+
+
+  private AuthenticationMethod getAuthMethod(final boolean isSecureClient,
+                                             final boolean isSecureServer,
+                                             final boolean useToken
+                                             
+      ) throws Exception {
     TestTokenSecretManager sm = new TestTokenSecretManager();
     Server server = new RPC.Builder(conf).setProtocol(TestSaslProtocol.class)
         .setInstance(new TestSaslImpl()).setBindAddress(ADDRESS).setPort(0)
         .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();      
-    if (secureServer) {
+    if (isSecureServer) {
       server.enableSecurity();
     } else {
       server.disableSecurity();
@@ -474,30 +549,36 @@ public class TestSaslRPC {
 
     final UserGroupInformation current = UserGroupInformation.getCurrentUser();
     final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-    TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
-        .getUserName()));
-    Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
-        sm);
-    SecurityUtil.setTokenService(token, addr);
-    current.addToken(token);
+    if (useToken) {
+      TestTokenIdentifier tokenId = new TestTokenIdentifier(
+          new Text(current.getUserName()));
+      Token<TestTokenIdentifier> token =
+          new Token<TestTokenIdentifier>(tokenId, sm);
+      SecurityUtil.setTokenService(token, addr);
+      current.addToken(token);
+    }
 
-    current.doAs(new PrivilegedExceptionAction<Object>() {
-      @Override
-      public Object run() throws IOException {
-        TestSaslProtocol proxy = null;
-        try {
-          proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
-              TestSaslProtocol.versionID, addr, conf);
-          Assert.assertEquals(AuthenticationMethod.TOKEN, proxy.getAuthMethod());
-        } finally {
-          if (proxy != null) {
-            RPC.stopProxy(proxy);
+    conf.set(HADOOP_SECURITY_AUTHENTICATION, isSecureClient ? "kerberos" : "simple");
+    UserGroupInformation.setConfiguration(conf);
+    try {
+      return current.doAs(new PrivilegedExceptionAction<AuthenticationMethod>() {
+        @Override
+        public AuthenticationMethod run() throws IOException {
+          TestSaslProtocol proxy = null;
+          try {
+            proxy = (TestSaslProtocol) RPC.getProxy(TestSaslProtocol.class,
+                TestSaslProtocol.versionID, addr, conf);
+            return proxy.getAuthMethod();
+          } finally {
+            if (proxy != null) {
+              RPC.stopProxy(proxy);
+            }
           }
         }
-        return null;
-      }
-    });
-    server.stop();
+      });
+    } finally {
+      server.stop();
+    }
   }
   
   public static void main(String[] args) throws Exception {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java

@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends HttpServerFunctionalTest {
         "/jmx?get=java.lang:type=Memory::"));
     LOG.info("/jmx RESULT: "+result);
     assertReFind("\"ERROR\"", result);
+
+    // test to get JSONP result
+    result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
+    LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
+    assertReFind("^mycallback1\\(\\{", result);
+    assertReFind("\\}\\);$", result);
+
+    // negative test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback2\\(\\{", result);
+    assertReFind("\"ERROR\"", result);
+    assertReFind("\\}\\);$", result);
+
+    // test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback3\\(\\{", result);
+    assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+    assertReFind("\"committed\"\\s*:", result);
+    assertReFind("\\}\\);$", result);
+
   }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java

@@ -18,9 +18,7 @@
 
 package org.apache.hadoop.metrics2.util;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
     // Counts start off zero
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException from empty window");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    
+    // Snapshot should be null if there are no entries.
+    assertNull(estimator.snapshot());
 
     // Count increment correctly by 1
     estimator.insert(1337);
     assertEquals(estimator.getCount(), 1);
     estimator.snapshot();
     assertEquals(estimator.getSampleCount(), 1);
+    
+    assertEquals(
+        "50.00 %ile +/- 5.00%: 1337\n" +
+        "75.00 %ile +/- 2.50%: 1337\n" +
+        "90.00 %ile +/- 1.00%: 1337\n" +
+        "95.00 %ile +/- 0.50%: 1337\n" +
+        "99.00 %ile +/- 0.10%: 1337", estimator.toString());
   }
 
   /**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
     estimator.clear();
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException for an empty window.");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    assertNull(estimator.snapshot());
   }
 
   /**

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java

@@ -142,16 +142,14 @@ public class TestPureJavaCrc32 {
       for (int j = 0; j < tables.length; j++) {
         final int[] t = tables[j];
         final StringBuilder b = new StringBuilder();
-        b.append(String.format("  static final int[] " + nameformat
-            + " = new int[] {", j));
+        b.append(String.format("    /* "+ nameformat +" */", j));
         for (int i = 0; i < t.length;) {
           b.append("\n    ");
           for(int k = 0; k < 4; k++) {
             b.append(String.format("0x%08X, ", t[i++]));
           }
         }
-        b.setCharAt(b.length() - 2, '\n');
-        s[j] = b.toString() + " };\n";
+        s[j] = b.toString();
       }
       return s;
     }
@@ -159,10 +157,23 @@ public class TestPureJavaCrc32 {
     @Override
     public String toString() {
       final StringBuilder b = new StringBuilder();
-      for(String s : toStrings(String.format("T%d_",
-          Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+      final String tableFormat = String.format("T%d_", 
+        Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+      final String startFormat = "  private static final int "+tableFormat+"_start = %d*256;";
+
+      for (int j = 0; j < tables.length; j++) {
+        b.append(String.format(startFormat, j, j));
+        b.append("\n");
+      }
+
+      b.append("  private static final int[] T = new int[] {");
+      for(String s : toStrings(tableFormat)) {
+        b.append("\n");
         b.append(s);
       }
+      b.setCharAt(b.length() - 2, '\n');
+      b.append(" };\n");
       return b.toString();
     }
 

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -147,6 +147,9 @@ Trunk (Unreleased)
     Block Pool Used, Block Pool Used(%) and Failed Volumes.
     (Brahma Reddy Battula via suresh)
 
+    HDFS-4052. BlockManager#invalidateWork should print log outside the lock.
+    (Jing Zhao via suresh)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -224,8 +227,6 @@ Trunk (Unreleased)
     HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from
     Command. (Jing Zhao via suresh)
 
-    HDFS-3678. Edit log files are never being purged from 2NN. (atm)
-
     HADOOP-8158. Interrupting hadoop fs -put from the command line
     causes a LeaseExpiredException. (daryn via harsh)
 
@@ -343,6 +344,8 @@ Release 2.0.3-alpha - Unreleased
     HDFS-3912. Detect and avoid stale datanodes for writes.
     (Jing Zhao via suresh)
 
+    HDFS-4059. Add number of stale DataNodes to metrics. (Jing Zhao via suresh)
+
   IMPROVEMENTS
   
     HDFS-3925. Prettify PipelineAck#toString() for printing to a log
@@ -350,8 +353,8 @@ Release 2.0.3-alpha - Unreleased
 
     HDFS-3939. NN RPC address cleanup. (eli)
 
-    HDFS-3373. Change DFSClient input stream socket cache to global static and add
-    a thread to cleanup expired cache entries. (John George via szetszwo)
+    HDFS-3373. Change DFSClient input stream socket cache to global static and
+    add a thread to cleanup expired cache entries. (John George via szetszwo)
 
     HDFS-3896. Add descriptions for dfs.namenode.rpc-address and
     dfs.namenode.servicerpc-address to hdfs-default.xml. (Jeff Lord via atm)
@@ -379,6 +382,34 @@ Release 2.0.3-alpha - Unreleased
     HDFS-4041. Hadoop HDFS Maven protoc calls must not depend on external
     sh script. (Chris Nauroth via suresh)
 
+    HADOOP-8911. CRLF characters in source and text files.
+    (Raja Aluri via suresh)
+
+    HDFS-4037. Rename the getReplication() method in BlockCollection to
+    getBlockReplication(). (szetszwo)
+
+    HDFS-4036. Remove "throws UnresolvedLinkException" from
+    FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo)
+
+    HDFS-2946. HA: Put a cap on the number of completed edits files retained
+    by the NN. (atm)
+
+    HDFS-4029. GenerationStamp should use an AtomicLong. (eli)
+
+    HDFS-4068. DatanodeID and DatanodeInfo member should be private. (eli)
+
+    HDFS-4073. Two minor improvements to FSDirectory.  (Jing Zhao via szetszwo)
+
+    HDFS-4074. Remove the unused default constructor from INode.  (Brandon Li
+    via szetszwo)
+
+    HDFS-4053. Increase the default block size. (eli)
+
+    HDFS-4088. Remove "throws QuotaExceededException" from an
+    INodeDirectoryWithQuota constructor. (szetszwo)
+
+    HDFS-4099. Clean up replication code and add more javadoc. (szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -428,6 +459,31 @@ Release 2.0.3-alpha - Unreleased
     HDFS-4021. Misleading error message when resources are low on the NameNode.
     (Christopher Conner via atm)
 
+    HDFS-4044. Duplicate ChecksumType definition in HDFS .proto files.
+    (Binglin Chang via suresh)
+
+    HDFS-4049. Fix hflush performance regression due to nagling delays
+    (todd)
+
+    HDFS-3678. Edit log files are never being purged from 2NN. (atm)
+
+    HDFS-4058. DirectoryScanner may fail with IOOB if the directory
+    scanning threads return out of volume order. (eli)
+
+    HDFS-3985. Add timeouts to TestMulitipleNNDataBlockScanner. (todd via eli)
+
+    HDFS-4061. TestBalancer and TestUnderReplicatedBlocks need timeouts. (eli)
+
+    HDFS-3997. OfflineImageViewer incorrectly passes value of imageVersion when
+    visiting IS_COMPRESSED element. (Mithun Radhakrishnan via atm)
+
+    HDFS-4055. TestAuditLogs is flaky. (Binglin Chang via eli)
+
+    HDFS-4072. On file deletion remove corresponding blocks pending
+    replications. (Jing Zhao via suresh)
+
+    HDFS-4022. Replication not happening for appended block. (Vinay via umamahesh)
+
 Release 2.0.2-alpha - 2012-09-07 
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/libhdfs.xml

@@ -1,110 +1,110 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
-          "http://forrest.apache.org/dtd/document-v20.dtd">
-
-<document>
-<header>
-<title>C API libhdfs</title>
-<meta name="http-equiv">Content-Type</meta>
-<meta name="content">text/html;</meta>
-<meta name="charset">utf-8</meta>
-</header>
-<body>
-<section>
-<title>Overview</title>
-
-<p>
-libhdfs is a JNI based C API for Hadoop's Distributed File System (HDFS).
-It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and
-the filesystem. libhdfs is part of the Hadoop distribution and comes 
-pre-compiled in ${HADOOP_PREFIX}/libhdfs/libhdfs.so .
-</p>
-
-</section>
-<section>
-<title>The APIs</title>
-
-<p>
-The libhdfs APIs are a subset of: <a href="api/org/apache/hadoop/fs/FileSystem.html" >hadoop fs APIs</a>.  
-</p>
-<p>
-The header file for libhdfs describes each API in detail and is available in ${HADOOP_PREFIX}/src/c++/libhdfs/hdfs.h
-</p>
-</section>
-<section>
-<title>A Sample Program</title>
-
-<source>
-#include "hdfs.h" 
-
-int main(int argc, char **argv) {
-
-    hdfsFS fs = hdfsConnect("default", 0);
-    const char* writePath = "/tmp/testfile.txt";
-    hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
-    if(!writeFile) {
-          fprintf(stderr, "Failed to open %s for writing!\n", writePath);
-          exit(-1);
-    }
-    char* buffer = "Hello, World!";
-    tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
-    if (hdfsFlush(fs, writeFile)) {
-           fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
-          exit(-1);
-    }
-   hdfsCloseFile(fs, writeFile);
-}
-</source>
-</section>
-
-<section>
-<title>How To Link With The Library</title>
-<p>
-See the Makefile for hdfs_test.c in the libhdfs source directory (${HADOOP_PREFIX}/src/c++/libhdfs/Makefile) or something like:<br />
-gcc above_sample.c -I${HADOOP_PREFIX}/src/c++/libhdfs -L${HADOOP_PREFIX}/libhdfs -lhdfs -o above_sample
-</p>
-</section>
-<section>
-<title>Common Problems</title>
-<p>
-The most common problem is the CLASSPATH is not set properly when calling a program that uses libhdfs. 
-Make sure you set it to all the Hadoop jars needed to run Hadoop itself. Currently, there is no way to 
-programmatically generate the classpath, but a good bet is to include all the jar files in ${HADOOP_PREFIX} 
-and ${HADOOP_PREFIX}/lib as well as the right configuration directory containing hdfs-site.xml
-</p>
-</section>
-<section>
-<title>Thread Safe</title>
-<p>libdhfs is thread safe.</p>
-<ul>
-<li>Concurrency and Hadoop FS "handles" 
-<br />The Hadoop FS implementation includes a FS handle cache which caches based on the URI of the 
-namenode along with the user connecting. So, all calls to hdfsConnect will return the same handle but 
-calls to hdfsConnectAsUser with different users will return different handles.  But, since HDFS client 
-handles are completely thread safe, this has no bearing on concurrency. 
-</li>
-<li>Concurrency and libhdfs/JNI 
-<br />The libhdfs calls to JNI should always be creating thread local storage, so (in theory), libhdfs 
-should be as thread safe as the underlying calls to the Hadoop FS.
-</li>
-</ul>
-</section>
-</body>
-</document>
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+          "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+<header>
+<title>C API libhdfs</title>
+<meta name="http-equiv">Content-Type</meta>
+<meta name="content">text/html;</meta>
+<meta name="charset">utf-8</meta>
+</header>
+<body>
+<section>
+<title>Overview</title>
+
+<p>
+libhdfs is a JNI based C API for Hadoop's Distributed File System (HDFS).
+It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and
+the filesystem. libhdfs is part of the Hadoop distribution and comes 
+pre-compiled in ${HADOOP_PREFIX}/libhdfs/libhdfs.so .
+</p>
+
+</section>
+<section>
+<title>The APIs</title>
+
+<p>
+The libhdfs APIs are a subset of: <a href="api/org/apache/hadoop/fs/FileSystem.html" >hadoop fs APIs</a>.  
+</p>
+<p>
+The header file for libhdfs describes each API in detail and is available in ${HADOOP_PREFIX}/src/c++/libhdfs/hdfs.h
+</p>
+</section>
+<section>
+<title>A Sample Program</title>
+
+<source>
+#include "hdfs.h" 
+
+int main(int argc, char **argv) {
+
+    hdfsFS fs = hdfsConnect("default", 0);
+    const char* writePath = "/tmp/testfile.txt";
+    hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_CREAT, 0, 0, 0);
+    if(!writeFile) {
+          fprintf(stderr, "Failed to open %s for writing!\n", writePath);
+          exit(-1);
+    }
+    char* buffer = "Hello, World!";
+    tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer)+1);
+    if (hdfsFlush(fs, writeFile)) {
+           fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
+          exit(-1);
+    }
+   hdfsCloseFile(fs, writeFile);
+}
+</source>
+</section>
+
+<section>
+<title>How To Link With The Library</title>
+<p>
+See the Makefile for hdfs_test.c in the libhdfs source directory (${HADOOP_PREFIX}/src/c++/libhdfs/Makefile) or something like:<br />
+gcc above_sample.c -I${HADOOP_PREFIX}/src/c++/libhdfs -L${HADOOP_PREFIX}/libhdfs -lhdfs -o above_sample
+</p>
+</section>
+<section>
+<title>Common Problems</title>
+<p>
+The most common problem is the CLASSPATH is not set properly when calling a program that uses libhdfs. 
+Make sure you set it to all the Hadoop jars needed to run Hadoop itself. Currently, there is no way to 
+programmatically generate the classpath, but a good bet is to include all the jar files in ${HADOOP_PREFIX} 
+and ${HADOOP_PREFIX}/lib as well as the right configuration directory containing hdfs-site.xml
+</p>
+</section>
+<section>
+<title>Thread Safe</title>
+<p>libdhfs is thread safe.</p>
+<ul>
+<li>Concurrency and Hadoop FS "handles" 
+<br />The Hadoop FS implementation includes a FS handle cache which caches based on the URI of the 
+namenode along with the user connecting. So, all calls to hdfsConnect will return the same handle but 
+calls to hdfsConnectAsUser with different users will return different handles.  But, since HDFS client 
+handles are completely thread safe, this has no bearing on concurrency. 
+</li>
+<li>Concurrency and libhdfs/JNI 
+<br />The libhdfs calls to JNI should always be creating thread local storage, so (in theory), libhdfs 
+should be as thread safe as the underlying calls to the Hadoop FS.
+</li>
+</ul>
+</section>
+</body>
+</document>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
 public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_BLOCK_SIZE_KEY = "dfs.blocksize";
-  public static final long    DFS_BLOCK_SIZE_DEFAULT = 64*1024*1024;
+  public static final long    DFS_BLOCK_SIZE_DEFAULT = 128*1024*1024;
   public static final String  DFS_REPLICATION_KEY = "dfs.replication";
   public static final short   DFS_REPLICATION_DEFAULT = 3;
   public static final String  DFS_STREAM_BUFFER_SIZE_KEY = "dfs.stream-buffer-size";
@@ -162,6 +162,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT = 2;
   public static final String  DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY = "dfs.namenode.num.extra.edits.retained";
   public static final int     DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT = 1000000; //1M
+  public static final String  DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY = "dfs.namenode.max.extra.edits.segments.retained";
+  public static final int     DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_DEFAULT = 10000; // 10k
   public static final String  DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY = "dfs.namenode.min.supported.datanode.version";
   public static final String  DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT = "3.0.0-SNAPSHOT";
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java

@@ -37,12 +37,12 @@ import org.apache.hadoop.classification.InterfaceStability;
 public class DatanodeID implements Comparable<DatanodeID> {
   public static final DatanodeID[] EMPTY_ARRAY = {};
 
-  protected String ipAddr;     // IP address
-  protected String hostName;   // hostname
-  protected String storageID;  // unique per cluster storageID
-  protected int xferPort;      // data streaming port
-  protected int infoPort;      // info server port
-  protected int ipcPort;       // IPC server port
+  private String ipAddr;     // IP address
+  private String hostName;   // hostname
+  private String storageID;  // unique per cluster storageID
+  private int xferPort;      // data streaming port
+  private int infoPort;      // info server port
+  private int ipcPort;       // IPC server port
 
   public DatanodeID(DatanodeID from) {
     this(from.getIpAddr(),

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java

@@ -37,13 +37,13 @@ import org.apache.hadoop.util.Time;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class DatanodeInfo extends DatanodeID implements Node {
-  protected long capacity;
-  protected long dfsUsed;
-  protected long remaining;
-  protected long blockPoolUsed;
-  protected long lastUpdate;
-  protected int xceiverCount;
-  protected String location = NetworkTopology.DEFAULT_RACK;
+  private long capacity;
+  private long dfsUsed;
+  private long remaining;
+  private long blockPoolUsed;
+  private long lastUpdate;
+  private int xceiverCount;
+  private String location = NetworkTopology.DEFAULT_RACK;
   
   // Datanode administrative states
   public enum AdminStates {
@@ -81,8 +81,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
     this.lastUpdate = from.getLastUpdate();
     this.xceiverCount = from.getXceiverCount();
     this.location = from.getNetworkLocation();
-    this.adminState = from.adminState;
-    this.hostName = from.hostName;
+    this.adminState = from.getAdminState();
   }
 
   public DatanodeInfo(DatanodeID nodeID) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java

@@ -24,9 +24,9 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BaseHeaderProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ChecksumProto;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ChecksumProto.ChecksumType;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ClientOperationHeaderProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ChecksumTypeProto;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
@@ -52,7 +52,7 @@ public abstract class DataTransferProtoUtil {
   }
 
   public static ChecksumProto toProto(DataChecksum checksum) {
-    ChecksumType type = ChecksumType.valueOf(checksum.getChecksumType().name());
+    ChecksumTypeProto type = ChecksumTypeProto.valueOf(checksum.getChecksumType().name());
     if (type == null) {
       throw new IllegalArgumentException(
           "Can't convert checksum to protobuf: " + checksum);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java

@@ -53,14 +53,8 @@ public class PacketReceiver implements Closeable {
   private final boolean useDirectBuffers;
 
   /**
-   * Internal buffer for reading the length prefixes at the start of
-   * the packet.
-   */
-  private final ByteBuffer lengthPrefixBuf = ByteBuffer.allocate(
-      PacketHeader.PKT_LENGTHS_LEN);
-
-  /**
-   * The entirety of the most recently read packet, excepting the
+   * The entirety of the most recently read packet.
+   * The first PKT_LENGTHS_LEN bytes of this buffer are the
    * length prefixes.
    */
   private ByteBuffer curPacketBuf = null;
@@ -82,6 +76,7 @@ public class PacketReceiver implements Closeable {
   
   public PacketReceiver(boolean useDirectBuffers) {
     this.useDirectBuffers = useDirectBuffers;
+    reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN);
   }
 
   public PacketHeader getHeader() {
@@ -133,11 +128,12 @@ public class PacketReceiver implements Closeable {
     //            checksums were not requested
     // DATA       the actual block data
     Preconditions.checkState(curHeader == null || !curHeader.isLastPacketInBlock());
-    
-    lengthPrefixBuf.clear();
-    doReadFully(ch, in, lengthPrefixBuf);
-    lengthPrefixBuf.flip();
-    int payloadLen = lengthPrefixBuf.getInt();
+
+    curPacketBuf.clear();
+    curPacketBuf.limit(PacketHeader.PKT_LENGTHS_LEN);
+    doReadFully(ch, in, curPacketBuf);
+    curPacketBuf.flip();
+    int payloadLen = curPacketBuf.getInt();
     
     if (payloadLen < Ints.BYTES) {
       // The "payload length" includes its own length. Therefore it
@@ -146,7 +142,7 @@ public class PacketReceiver implements Closeable {
           payloadLen);
     }
     int dataPlusChecksumLen = payloadLen - Ints.BYTES;
-    int headerLen = lengthPrefixBuf.getShort();
+    int headerLen = curPacketBuf.getShort();
     if (headerLen < 0) {
       throw new IOException("Invalid header length " + headerLen);
     }
@@ -166,13 +162,17 @@ public class PacketReceiver implements Closeable {
 
     // Make sure we have space for the whole packet, and
     // read it.
-    reallocPacketBuf(dataPlusChecksumLen + headerLen);
+    reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN +
+        dataPlusChecksumLen + headerLen);
     curPacketBuf.clear();
-    curPacketBuf.limit(dataPlusChecksumLen + headerLen);
+    curPacketBuf.position(PacketHeader.PKT_LENGTHS_LEN);
+    curPacketBuf.limit(PacketHeader.PKT_LENGTHS_LEN +
+        dataPlusChecksumLen + headerLen);
     doReadFully(ch, in, curPacketBuf);
     curPacketBuf.flip();
+    curPacketBuf.position(PacketHeader.PKT_LENGTHS_LEN);
 
-    // Extract the header from the front of the buffer.
+    // Extract the header from the front of the buffer (after the length prefixes)
     byte[] headerBuf = new byte[headerLen];
     curPacketBuf.get(headerBuf);
     if (curHeader == null) {
@@ -197,10 +197,6 @@ public class PacketReceiver implements Closeable {
   public void mirrorPacketTo(DataOutputStream mirrorOut) throws IOException {
     Preconditions.checkState(!useDirectBuffers,
         "Currently only supported for non-direct buffers");
-    assert lengthPrefixBuf.capacity() == PacketHeader.PKT_LENGTHS_LEN;
-    mirrorOut.write(lengthPrefixBuf.array(),
-        lengthPrefixBuf.arrayOffset(),
-        lengthPrefixBuf.capacity());
     mirrorOut.write(curPacketBuf.array(),
         curPacketBuf.arrayOffset(),
         curPacketBuf.remaining());
@@ -223,23 +219,36 @@ public class PacketReceiver implements Closeable {
 
   private void reslicePacket(
       int headerLen, int checksumsLen, int dataLen) {
+    // Packet structure (refer to doRead() for details):
+    //   PLEN    HLEN      HEADER     CHECKSUMS  DATA
+    //   32-bit  16-bit   <protobuf>  <variable length>
+    //   |--- lenThroughHeader ----|
+    //   |----------- lenThroughChecksums   ----|
+    //   |------------------- lenThroughData    ------| 
+    int lenThroughHeader = PacketHeader.PKT_LENGTHS_LEN + headerLen;
+    int lenThroughChecksums = lenThroughHeader + checksumsLen;
+    int lenThroughData = lenThroughChecksums + dataLen;
+
     assert dataLen >= 0 : "invalid datalen: " + dataLen;
-    
-    assert curPacketBuf.position() == headerLen;
-    assert checksumsLen + dataLen == curPacketBuf.remaining() :
+    assert curPacketBuf.position() == lenThroughHeader;
+    assert curPacketBuf.limit() == lenThroughData :
       "headerLen= " + headerLen + " clen=" + checksumsLen + " dlen=" + dataLen +
       " rem=" + curPacketBuf.remaining();
-    
-    curPacketBuf.position(headerLen);
-    curPacketBuf.limit(headerLen + checksumsLen);
+
+    // Slice the checksums.
+    curPacketBuf.position(lenThroughHeader);
+    curPacketBuf.limit(lenThroughChecksums);
     curChecksumSlice = curPacketBuf.slice();
 
-    curPacketBuf.position(headerLen + checksumsLen);
-    curPacketBuf.limit(headerLen + checksumsLen + dataLen);
+    // Slice the data.
+    curPacketBuf.position(lenThroughChecksums);
+    curPacketBuf.limit(lenThroughData);
     curDataSlice = curPacketBuf.slice();
     
+    // Reset buffer to point to the entirety of the packet (including
+    // length prefixes)
     curPacketBuf.position(0);
-    curPacketBuf.limit(headerLen + checksumsLen + dataLen);
+    curPacketBuf.limit(lenThroughData);
   }
 
   
@@ -258,12 +267,21 @@ public class PacketReceiver implements Closeable {
     // one.
     if (curPacketBuf == null ||
         curPacketBuf.capacity() < atLeastCapacity) {
-      returnPacketBufToPool();
+      ByteBuffer newBuf;
       if (useDirectBuffers) {
-        curPacketBuf = bufferPool.getBuffer(atLeastCapacity);
+        newBuf = bufferPool.getBuffer(atLeastCapacity);
       } else {
-        curPacketBuf = ByteBuffer.allocate(atLeastCapacity);
+        newBuf = ByteBuffer.allocate(atLeastCapacity);
       }
+      // If reallocing an existing buffer, copy the old packet length
+      // prefixes over
+      if (curPacketBuf != null) {
+        curPacketBuf.flip();
+        newBuf.put(curPacketBuf);
+      }
+      
+      returnPacketBufToPool();
+      curPacketBuf = newBuf;
     }
   }
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java

@@ -19,12 +19,14 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.io.IOException;
 
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.ContentSummary;
 
 /** 
  * This interface is used by the block manager to expose a
  * few characteristics of a collection of Block/BlockUnderConstruction.
  */
+@InterfaceAudience.Private
 public interface BlockCollection {
   /**
    * Get the last block of the collection.
@@ -56,7 +58,7 @@ public interface BlockCollection {
    * Get block replication for the collection 
    * @return block replication value
    */
-  public short getReplication();
+  public short getBlockReplication();
 
   /**
    * Get the name of the collection.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java

@@ -73,7 +73,7 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
    * @param from BlockInfo to copy from.
    */
   protected BlockInfo(BlockInfo from) {
-    this(from, from.bc.getReplication());
+    this(from, from.bc.getBlockReplication());
     this.bc = from.bc;
   }
 
@@ -335,7 +335,7 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
       BlockUCState s, DatanodeDescriptor[] targets) {
     if(isComplete()) {
       return new BlockInfoUnderConstruction(
-          this, getBlockCollection().getReplication(), s, targets);
+          this, getBlockCollection().getBlockReplication(), s, targets);
     }
     // the block is already under construction
     BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import static org.apache.hadoop.util.ExitUtil.terminate;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
@@ -49,14 +51,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
-import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo;
-
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.namenode.FSClusterStats;
@@ -288,7 +287,7 @@ public class BlockManager {
   }
 
   private static BlockTokenSecretManager createBlockTokenSecretManager(
-      final Configuration conf) throws IOException {
+      final Configuration conf) {
     final boolean isEnabled = conf.getBoolean(
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@@ -624,7 +623,10 @@ public class BlockManager {
     blocksMap.replaceBlock(ucBlock);
 
     // Remove block from replication queue.
-    updateNeededReplications(oldBlock, 0, 0);
+    NumberReplicas replicas = countNodes(ucBlock);
+    neededReplications.remove(ucBlock, replicas.liveReplicas(),
+        replicas.decommissionedReplicas(), getReplication(ucBlock));
+    pendingReplications.remove(ucBlock);
 
     // remove this block from the list of pending blocks to be deleted. 
     for (DatanodeDescriptor dd : targets) {
@@ -997,7 +999,7 @@ public class BlockManager {
 
     // Add this replica to corruptReplicas Map
     corruptReplicas.addToCorruptReplicasMap(b.corrupted, node, b.reason);
-    if (countNodes(b.stored).liveReplicas() >= bc.getReplication()) {
+    if (countNodes(b.stored).liveReplicas() >= bc.getBlockReplication()) {
       // the block is over-replicated so invalidate the replicas immediately
       invalidateBlock(b, node);
     } else if (namesystem.isPopulatingReplQueues()) {
@@ -1135,7 +1137,7 @@ public class BlockManager {
               continue;
             }
 
-            requiredReplication = bc.getReplication();
+            requiredReplication = bc.getBlockReplication();
 
             // get a source data-node
             containingNodes = new ArrayList<DatanodeDescriptor>();
@@ -1221,7 +1223,7 @@ public class BlockManager {
             neededReplications.decrementReplicationIndex(priority);
             continue;
           }
-          requiredReplication = bc.getReplication();
+          requiredReplication = bc.getBlockReplication();
 
           // do not schedule more if enough replicas is already pending
           NumberReplicas numReplicas = countNodes(block);
@@ -1260,7 +1262,7 @@ public class BlockManager {
           // Move the block-replication into a "pending" state.
           // The reason we use 'pending' is so we can retry
           // replications that fail after an appropriate amount of time.
-          pendingReplications.add(block, targets.length);
+          pendingReplications.increment(block, targets.length);
           if(NameNode.stateChangeLog.isDebugEnabled()) {
             NameNode.stateChangeLog.debug(
                 "BLOCK* block " + block
@@ -1306,8 +1308,11 @@ public class BlockManager {
 
   /**
    * Choose target datanodes according to the replication policy.
-   * @throws IOException if the number of targets < minimum replication.
-   * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, HashMap, long)
+   * 
+   * @throws IOException
+   *           if the number of targets < minimum replication.
+   * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor,
+   *      List, boolean, HashMap, long)
    */
   public DatanodeDescriptor[] chooseTarget(final String src,
       final int numOfReplicas, final DatanodeDescriptor client,
@@ -1811,7 +1816,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
 
   /**
    * Queue the given reported block for later processing in the
-   * standby node. {@see PendingDataNodeMessages}.
+   * standby node. @see PendingDataNodeMessages.
    * @param reason a textual reason to report in the debug logs
    */
   private void queueReportedBlock(DatanodeDescriptor dn, Block block,
@@ -1976,14 +1981,15 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
   }
   
   /**
-   * Faster version of {@link addStoredBlock()}, intended for use with 
-   * initial block report at startup.  If not in startup safe mode, will
-   * call standard addStoredBlock().
-   * Assumes this method is called "immediately" so there is no need to
-   * refresh the storedBlock from blocksMap.
-   * Doesn't handle underReplication/overReplication, or worry about
+   * Faster version of
+   * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)}
+   * , intended for use with initial block report at startup. If not in startup
+   * safe mode, will call standard addStoredBlock(). Assumes this method is
+   * called "immediately" so there is no need to refresh the storedBlock from
+   * blocksMap. Doesn't handle underReplication/overReplication, or worry about
    * pendingReplications or corruptReplicas, because it's in startup safe mode.
    * Doesn't log every block, because there are typically millions of them.
+   * 
    * @throws IOException
    */
   private void addStoredBlockImmediate(BlockInfo storedBlock,
@@ -2089,7 +2095,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     }
 
     // handle underReplication/overReplication
-    short fileReplication = bc.getReplication();
+    short fileReplication = bc.getBlockReplication();
     if (!isNeededReplication(storedBlock, fileReplication, numCurrentReplica)) {
       neededReplications.remove(storedBlock, numCurrentReplica,
           num.decommissionedReplicas(), fileReplication);
@@ -2228,7 +2234,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
       return MisReplicationResult.UNDER_CONSTRUCTION;
     }
     // calculate current replication
-    short expectedReplication = bc.getReplication();
+    short expectedReplication = bc.getBlockReplication();
     NumberReplicas num = countNodes(block);
     int numCurrentReplica = num.liveReplicas();
     // add to under-replicated queue if need to be
@@ -2505,7 +2511,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     //
     // Modify the blocks->datanode map and node's map.
     //
-    pendingReplications.remove(block);
+    pendingReplications.decrement(block);
     processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED,
         delHintNode);
   }
@@ -2641,7 +2647,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
   }
 
   /** 
-   * Simpler, faster form of {@link countNodes()} that only returns the number
+   * Simpler, faster form of {@link #countNodes(Block)} that only returns the number
    * of live nodes.  If in startup safemode (or its 30-sec extension period),
    * then it gains speed by ignoring issues of excess replicas or nodes
    * that are decommissioned or in process of becoming decommissioned.
@@ -2699,7 +2705,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     while(it.hasNext()) {
       final Block block = it.next();
       BlockCollection bc = blocksMap.getBlockCollection(block);
-      short expectedReplication = bc.getReplication();
+      short expectedReplication = bc.getBlockReplication();
       NumberReplicas num = countNodes(block);
       int numCurrentReplica = num.liveReplicas();
       if (numCurrentReplica > expectedReplication) {
@@ -2790,6 +2796,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     addToInvalidates(block);
     corruptReplicas.removeFromCorruptReplicasMap(block);
     blocksMap.removeBlock(block);
+    // Remove the block from pendingReplications
+    pendingReplications.remove(block);
     if (postponedMisreplicatedBlocks.remove(block)) {
       postponedMisreplicatedBlocksCount--;
     }
@@ -2824,28 +2832,32 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     }
   }
 
-  public void checkReplication(Block block, short numExpectedReplicas) {
-    // filter out containingNodes that are marked for decommission.
-    NumberReplicas number = countNodes(block);
-    if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) { 
-      neededReplications.add(block,
-                             number.liveReplicas(),
-                             number.decommissionedReplicas(),
-                             numExpectedReplicas);
-      return;
-    }
-    if (number.liveReplicas() > numExpectedReplicas) {
-      processOverReplicatedBlock(block, numExpectedReplicas, null, null);
+  /**
+   * Check replication of the blocks in the collection.
+   * If any block is needed replication, insert it into the replication queue.
+   * Otherwise, if the block is more than the expected replication factor,
+   * process it as an over replicated block.
+   */
+  public void checkReplication(BlockCollection bc) {
+    final short expected = bc.getBlockReplication();
+    for (Block block : bc.getBlocks()) {
+      final NumberReplicas n = countNodes(block);
+      if (isNeededReplication(block, expected, n.liveReplicas())) { 
+        neededReplications.add(block, n.liveReplicas(),
+            n.decommissionedReplicas(), expected);
+      } else if (n.liveReplicas() > expected) {
+        processOverReplicatedBlock(block, expected, null, null);
+      }
     }
   }
 
-  /* get replication factor of a block */
+  /** 
+   * @return 0 if the block is not found;
+   *         otherwise, return the replication factor of the block.
+   */
   private int getReplication(Block block) {
-    BlockCollection bc = blocksMap.getBlockCollection(block);
-    if (bc == null) { // block does not belong to any file
-      return 0;
-    }
-    return bc.getReplication();
+    final BlockCollection bc = blocksMap.getBlockCollection(block);
+    return bc == null? 0: bc.getBlockReplication();
   }
 
 
@@ -2856,6 +2868,9 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
    * @return number of blocks scheduled for removal during this iteration.
    */
   private int invalidateWorkForOneNode(String nodeId) {
+    final List<Block> toInvalidate;
+    final DatanodeDescriptor dn;
+    
     namesystem.writeLock();
     try {
       // blocks should not be replicated or removed if safe mode is on
@@ -2865,10 +2880,23 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
       }
       // get blocks to invalidate for the nodeId
       assert nodeId != null;
-      return invalidateBlocks.invalidateWork(nodeId);
+      dn = datanodeManager.getDatanode(nodeId);
+      if (dn == null) {
+        invalidateBlocks.remove(nodeId);
+        return 0;
+      }
+      toInvalidate = invalidateBlocks.invalidateWork(nodeId, dn);
+      if (toInvalidate == null) {
+        return 0;
+      }
     } finally {
       namesystem.writeUnlock();
     }
+    if (NameNode.stateChangeLog.isInfoEnabled()) {
+      NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName()
+          + ": ask " + dn + " to delete " + toInvalidate);
+    }
+    return toInvalidate.size();
   }
 
   boolean blockHasEnoughRacks(Block b) {
@@ -2904,12 +2932,12 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     return enoughRacks;
   }
 
-  boolean isNeededReplication(Block b, int expectedReplication, int curReplicas) {
-    if ((curReplicas >= expectedReplication) && (blockHasEnoughRacks(b))) {
-      return false;
-    } else {
-      return true;
-    }
+  /**
+   * A block needs replication if the number of replicas is less than expected
+   * or if it does not have enough racks.
+   */
+  private boolean isNeededReplication(Block b, int expected, int current) {
+    return current < expected || !blockHasEnoughRacks(b);
   }
   
   public long getMissingBlocksCount() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java

@@ -276,11 +276,11 @@ public class DatanodeDescriptor extends DatanodeInfo {
   }
 
   public void resetBlocks() {
-    this.capacity = 0;
-    this.remaining = 0;
-    this.blockPoolUsed = 0;
-    this.dfsUsed = 0;
-    this.xceiverCount = 0;
+    setCapacity(0);
+    setRemaining(0);
+    setBlockPoolUsed(0);
+    setDfsUsed(0);
+    setXceiverCount(0);
     this.blockList = null;
     this.invalidateBlocks.clear();
     this.volumeFailures = 0;
@@ -303,15 +303,15 @@ public class DatanodeDescriptor extends DatanodeInfo {
    */
   public void updateHeartbeat(long capacity, long dfsUsed, long remaining,
       long blockPoolUsed, int xceiverCount, int volFailures) {
-    this.capacity = capacity;
-    this.dfsUsed = dfsUsed;
-    this.remaining = remaining;
-    this.blockPoolUsed = blockPoolUsed;
-    this.lastUpdate = Time.now();
-    this.xceiverCount = xceiverCount;
+    setCapacity(capacity);
+    setRemaining(remaining);
+    setBlockPoolUsed(blockPoolUsed);
+    setDfsUsed(dfsUsed);
+    setXceiverCount(xceiverCount);
+    setLastUpdate(Time.now());    
     this.volumeFailures = volFailures;
     this.heartbeatedSinceFailover = true;
-    rollBlocksScheduled(lastUpdate);
+    rollBlocksScheduled(getLastUpdate());
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java

@@ -567,7 +567,7 @@ public class DatanodeManager {
   /**
    * Decommission the node if it is in exclude list.
    */
-  private void checkDecommissioning(DatanodeDescriptor nodeReg, String ipAddr) { 
+  private void checkDecommissioning(DatanodeDescriptor nodeReg) { 
     // If the registered node is in exclude list, then decommission it
     if (inExcludedHostsList(nodeReg)) {
       startDecommission(nodeReg);
@@ -713,7 +713,7 @@ public class DatanodeManager {
         
       // also treat the registration message as a heartbeat
       heartbeatManager.register(nodeS);
-      checkDecommissioning(nodeS, dnAddress);
+      checkDecommissioning(nodeS);
       return;
     } 
 
@@ -733,7 +733,7 @@ public class DatanodeManager {
       = new DatanodeDescriptor(nodeReg, NetworkTopology.DEFAULT_RACK);
     resolveNetworkLocation(nodeDescr);
     addDatanode(nodeDescr);
-    checkDecommissioning(nodeDescr, dnAddress);
+    checkDecommissioning(nodeDescr);
     
     // also treat the registration message as a heartbeat
     // no need to update its timestamp
@@ -885,7 +885,7 @@ public class DatanodeManager {
    * @return Return the current number of stale DataNodes (detected by
    * HeartbeatManager). 
    */
-  int getNumStaleNodes() {
+  public int getNumStaleNodes() {
     return this.numStaleNodes;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java

@@ -134,26 +134,7 @@ class InvalidateBlocks {
     return new ArrayList<String>(node2blocks.keySet());
   }
 
-  /** Invalidate work for the storage. */
-  int invalidateWork(final String storageId) {
-    final DatanodeDescriptor dn = datanodeManager.getDatanode(storageId);
-    if (dn == null) {
-      remove(storageId);
-      return 0;
-    }
-    final List<Block> toInvalidate = invalidateWork(storageId, dn);
-    if (toInvalidate == null) {
-      return 0;
-    }
-
-    if (NameNode.stateChangeLog.isInfoEnabled()) {
-      NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName()
-          + ": ask " + dn + " to delete " + toInvalidate);
-    }
-    return toInvalidate.size();
-  }
-
-  private synchronized List<Block> invalidateWork(
+  synchronized List<Block> invalidateWork(
       final String storageId, final DatanodeDescriptor dn) {
     final LightWeightHashSet<Block> set = node2blocks.get(storageId);
     if (set == null) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java

@@ -72,7 +72,7 @@ class PendingReplicationBlocks {
   /**
    * Add a block to the list of pending Replications
    */
-  void add(Block block, int numReplicas) {
+  void increment(Block block, int numReplicas) {
     synchronized (pendingReplications) {
       PendingBlockInfo found = pendingReplications.get(block);
       if (found == null) {
@@ -89,7 +89,7 @@ class PendingReplicationBlocks {
    * Decrement the number of pending replication requests
    * for this block.
    */
-  void remove(Block block) {
+  void decrement(Block block) {
     synchronized (pendingReplications) {
       PendingBlockInfo found = pendingReplications.get(block);
       if (found != null) {
@@ -104,6 +104,16 @@ class PendingReplicationBlocks {
     }
   }
 
+  /**
+   * Remove the record about the given block from pendingReplications.
+   * @param block The given block whose pending replication requests need to be
+   *              removed
+   */
+  void remove(Block block) {
+    synchronized (pendingReplications) {
+      pendingReplications.remove(block);
+    }
+  }
 
   public void clear() {
     synchronized (pendingReplications) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.common;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 
 /****************************************************************
@@ -35,7 +37,7 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
    */
   public static final long GRANDFATHER_GENERATION_STAMP = 0;
 
-  private volatile long genstamp;
+  private AtomicLong genstamp = new AtomicLong();
 
   /**
    * Create a new instance, initialized to FIRST_VALID_STAMP.
@@ -48,35 +50,36 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
    * Create a new instance, initialized to the specified value.
    */
   GenerationStamp(long stamp) {
-    this.genstamp = stamp;
+    genstamp.set(stamp);
   }
 
   /**
    * Returns the current generation stamp
    */
   public long getStamp() {
-    return this.genstamp;
+    return genstamp.get();
   }
 
   /**
    * Sets the current generation stamp
    */
   public void setStamp(long stamp) {
-    this.genstamp = stamp;
+    genstamp.set(stamp);
   }
 
   /**
    * First increments the counter and then returns the stamp 
    */
-  public synchronized long nextStamp() {
-    this.genstamp++;
-    return this.genstamp;
+  public long nextStamp() {
+    return genstamp.incrementAndGet();
   }
 
   @Override // Comparable
   public int compareTo(GenerationStamp that) {
-    return this.genstamp < that.genstamp ? -1 :
-           this.genstamp > that.genstamp ? 1 : 0;
+    long stamp1 = this.genstamp.get();
+    long stamp2 = that.genstamp.get();
+    return stamp1 < stamp2 ? -1 :
+           stamp1 > stamp2 ? 1 : 0;
   }
 
   @Override // Object
@@ -89,6 +92,7 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
 
   @Override // Object
   public int hashCode() {
-    return (int) (genstamp^(genstamp>>>32));
+    long stamp = genstamp.get();
+    return (int) (stamp^(stamp>>>32));
   }
 }