merge from trunk to branch MR-3902

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-3902@1400219 13f79535-47bb-0310-9956-ffa450edef68

dev-support/test-patch.sh

@@ -32,7 +32,7 @@ JENKINS=false
 PATCH_DIR=/tmp
 SUPPORT_DIR=/tmp
 BASEDIR=$(pwd)
-
+BUILD_NATIVE=true
 PS=${PS:-ps}
 AWK=${AWK:-awk}
 WGET=${WGET:-wget}
@@ -67,6 +67,7 @@ printUsage() {
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
   echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
   echo "--run-tests            Run all tests below the base directory"
+  echo "--build-native=<bool>  If true, then build native components (default 'true')"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -139,11 +140,18 @@ parseArgs() {
     --run-tests)
       RUN_TESTS=true
       ;;
+    --build-native=*)
+      BUILD_NATIVE=${i#*=}
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
     esac
   done
+  if [[ $BUILD_NATIVE == "true" ]] ; then
+    NATIVE_PROFILE=-Pnative
+    REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
+  fi
   if [ -z "$PATCH_OR_DEFECT" ]; then
     printUsage
     exit 1
@@ -437,8 +445,8 @@ checkJavacWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
-  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Pnative -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
+  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
+  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
@@ -707,8 +715,8 @@ runTests () {
       if [[ $building_common -eq 0 ]]; then
           echo "  Building hadoop-common with -Pnative in order to provide \
 libhadoop.so to the hadoop-hdfs unit tests."
-          echo "  $MVN compile -Pnative -D${PROJECT_NAME}PatchProcess"
-          if ! $MVN compile -Pnative -D${PROJECT_NAME}PatchProcess; then
+          echo "  $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess"
+          if ! $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess; then
               JIRA_COMMENT="$JIRA_COMMENT
         {color:red}-1 core tests{color}.  Failed to build the native portion \
 of hadoop-common prior to running the unit tests in $ordered_modules"
@@ -719,8 +727,8 @@ of hadoop-common prior to running the unit tests in $ordered_modules"
   for module in $ordered_modules; do
     cd $module
     echo "  Running tests in $module"
-    echo "  $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess"
-    $MVN clean install -fn -Pnative -Drequire.test.libhadoop -D${PROJECT_NAME}PatchProcess
+    echo "  $MVN clean install -fn $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess"
+    $MVN clean install -fn $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess
     module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
     # With -fn mvn always exits with a 0 exit code.  Because of this we need to
     # find the errors instead of using the exit code.  We assume that if the build

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -158,7 +158,7 @@ public class KerberosAuthenticator implements Authenticator {
       conn.setRequestMethod(AUTH_HTTP_METHOD);
       conn.connect();
       
-      if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
+      if (conn.getRequestProperty(AUTHORIZATION) != null && conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
         LOG.debug("JDK performed authentication on our behalf.");
         // If the JDK already did the SPNEGO back-and-forth for
         // us, just pull out the token.

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/AuthenticatorTestCase.java

@@ -134,9 +134,11 @@ public abstract class AuthenticatorTestCase extends TestCase {
     try {
       URL url = new URL(getBaseURL());
       AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+      Assert.assertFalse(token.isSet());
       TestConnectionConfigurator connConf = new TestConnectionConfigurator();
       AuthenticatedURL aUrl = new AuthenticatedURL(authenticator, connConf);
       HttpURLConnection conn = aUrl.openConnection(url, token);
+      Assert.assertTrue(token.isSet());
       Assert.assertTrue(connConf.invoked);
       String tokenStr = token.toString();
       if (doPost) {

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java

@@ -43,6 +43,14 @@ public class TestKerberosAuthenticator extends AuthenticatorTestCase {
     _testAuthentication(new KerberosAuthenticator(), false);
   }
 
+  public void testFallbacktoPseudoAuthenticatorAnonymous() throws Exception {
+    Properties props = new Properties();
+    props.setProperty(AuthenticationFilter.AUTH_TYPE, "simple");
+    props.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "true");
+    setAuthenticationHandlerConfig(props);
+    _testAuthentication(new KerberosAuthenticator(), false);
+  }
+
   public void testNotAuthenticated() throws Exception {
     setAuthenticationHandlerConfig(getAuthenticationHandlerConfiguration());
     start();

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -72,8 +72,8 @@ Trunk (Unreleased)
     HADOOP-8360. empty-configuration.xml fails xml validation
     (Radim Kolar via harsh)
 
-    HADOOP-8367 Improve documentation of declaringClassProtocolName in rpc headers 
-                (Sanjay Radia)
+    HADOOP-8367 Improve documentation of declaringClassProtocolName in 
+    rpc headers. (Sanjay Radia)
 
     HADOOP-8415. Add getDouble() and setDouble() in
     org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
@@ -124,6 +124,11 @@ Trunk (Unreleased)
 
     HADOOP-8910. Add examples to GlobExpander#expand method. (suresh)
 
+    HADOOP-8920. Add more javadoc to metrics2 related classes. (suresh)
+
+    HADOOP-8776. Provide an option in test-patch that can enable/disable
+    compiling native code. (Chris Nauroth via suresh)
+
   BUG FIXES
 
     HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
@@ -316,11 +321,23 @@ Release 2.0.3-alpha - Unreleased
 
     HADOOP-8784. Improve IPC.Client's token use (daryn)
 
+    HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+
+    HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
+    javascript in browser dashboard (Damien Hardy via bobby)
+
+    HADOOP-8931. Add Java version to startup message. (eli)
+
+    HADOOP-8925. Remove the packaging. (eli)
+
   OPTIMIZATIONS
 
     HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
     via atm)
 
+    HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+    data (Gopal V via bobby)
+
   BUG FIXES
 
     HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to
@@ -351,10 +368,15 @@ Release 2.0.3-alpha - Unreleased
     webhdfs filesystem and fsck to fail when security is on.
     (Arpit Gupta via suresh)
 
-
     HADOOP-8901. GZip and Snappy support may not work without unversioned
     libraries (Colin Patrick McCabe via todd)
 
+    HADOOP-8883. Anonymous fallback in KerberosAuthenticator is broken.
+    (rkanter via tucu)
+
+    HADOOP-8900. BuiltInGzipDecompressor throws IOException - stored gzip size
+    doesn't match decompressed size. (Slavik Krassovsky via suresh)
+
 Release 2.0.2-alpha - 2012-09-07 
 
   INCOMPATIBLE CHANGES
@@ -1053,6 +1075,12 @@ Release 0.23.5 - UNRELEASED
 
   IMPROVEMENTS
 
+    HADOOP-8923. JNI-based user-group mapping modules can be too chatty on 
+    lookup failures. (Kihwal Lee via suresh)
+
+    HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via
+    bobby)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java

@@ -387,7 +387,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
       copyBytesToLocal(n);       // modifies userBufLen, etc.
       if (localBufOff >= 4) {    // should be strictly ==
         long inputSize = readUIntLE(localBuf, 0);
-        if (inputSize != (inflater.getBytesWritten() & 0xffffffff)) {
+        if (inputSize != (inflater.getBytesWritten() & 0xffffffffL)) {
           throw new IOException(
             "stored gzip size doesn't match decompressed size");
         }
@@ -571,7 +571,7 @@ public class BuiltInGzipDecompressor implements Decompressor {
     return ((((long)(b[off+3] & 0xff) << 24) |
              ((long)(b[off+2] & 0xff) << 16) |
              ((long)(b[off+1] & 0xff) <<  8) |
-             ((long)(b[off]   & 0xff)      )) & 0xffffffff);
+             ((long)(b[off]   & 0xff)      )) & 0xffffffffL);
   }
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java

@@ -113,12 +113,17 @@ import org.codehaus.jackson.JsonGenerator;
  *  All other objects will be converted to a string and output as such.
  *  
  *  The bean's name and modelerType will be returned for all beans.
+ *
+ *  Optional paramater "callback" should be used to deliver JSONP response.
+ *  
  */
 public class JMXJsonServlet extends HttpServlet {
   private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
 
   private static final long serialVersionUID = 1L;
 
+  private static final String CALLBACK_PARAM = "callback";
+
   /**
    * MBean server.
    */
@@ -154,11 +159,22 @@ public class JMXJsonServlet extends HttpServlet {
         return;
       }
       JsonGenerator jg = null;
+      String jsonpcb = null;
+      PrintWriter writer = null;
       try {
-        response.setContentType("application/json; charset=utf8");
+        writer = response.getWriter();
+ 
+        // "callback" parameter implies JSONP outpout
+        jsonpcb = request.getParameter(CALLBACK_PARAM);
+        if (jsonpcb != null) {
+          response.setContentType("application/javascript; charset=utf8");
+          writer.write(jsonpcb + "(");
+        } else {
+          response.setContentType("application/json; charset=utf8");
+        }
 
-        PrintWriter writer = response.getWriter();
         jg = jsonFactory.createJsonGenerator(writer);
+        jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
         jg.useDefaultPrettyPrinter();
         jg.writeStartObject();
 
@@ -188,6 +204,12 @@ public class JMXJsonServlet extends HttpServlet {
         if (jg != null) {
           jg.close();
         }
+        if (jsonpcb != null) {
+           writer.write(");");
+        }
+        if (writer != null) {
+          writer.close();
+        }
       }
     } catch (IOException e) {
       LOG.error("Caught an exception while processing JMX request", e);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsCollector.java

@@ -30,14 +30,14 @@ public interface MetricsCollector {
   /**
    * Add a metrics record
    * @param name  of the record
-   * @return  a metrics record builder for the record
+   * @return  a {@link MetricsRecordBuilder} for the record {@code name}
    */
   public MetricsRecordBuilder addRecord(String name);
 
   /**
    * Add a metrics record
    * @param info  of the record
-   * @return  a metrics record builder for the record
+   * @return  a {@link MetricsRecordBuilder} for metrics {@code info}
    */
   public MetricsRecordBuilder addRecord(MetricsInfo info);
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java

@@ -22,7 +22,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics filter interface
+ * The metrics filter interface. The MetricsFilter objects can be used either to
+ * filter the metrics from {@link MetricsSource}s or to filter metrics per
+ * {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsInfo.java

@@ -20,19 +20,24 @@ package org.apache.hadoop.metrics2;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.annotation.Metric;
 
 /**
- * Interface to provide immutable meta info for metrics
+ * Interface to provide immutable metainfo for metrics.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface MetricsInfo {
   /**
+   * Typically name corresponds to annotation {@link Metric#value()} or
+   * the name of the class.
    * @return the name of the metric/tag
    */
   String name();
 
   /**
+   * Typically the description corresponds to annotation {@link Metric#about()}
+   * or the name of the class.
    * @return the description of the metric/tag
    */
   String description();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecord.java

@@ -36,17 +36,17 @@ public interface MetricsRecord {
   long timestamp();
 
   /**
-   * @return the record name
+   * @return the metrics record name
    */
   String name();
 
   /**
-   * @return the description of the record
+   * @return the description of the metrics record
    */
   String description();
 
   /**
-   * @return the context name of the record
+   * @return the context name of the metrics record
    */
   String context();
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsRecordBuilder.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceStability.Evolving
 public abstract class MetricsRecordBuilder {
   /**
-   * Add a metrics tag
+   * Add a metrics value with metrics information
    * @param info  metadata of the tag
    * @param value of the tag
    * @return self

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSink.java

@@ -22,7 +22,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics sink interface
+ * The metrics sink interface. <p>
+ * Implementations of this interface consume the {@link MetricsRecord} generated
+ * from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ * periodically pushes the {@link MetricsRecord} to the sink using
+ * {@link #putMetrics(MetricsRecord)} method.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSource.java

@@ -22,13 +22,15 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics source interface
+ * The source of metrics information. It generates and updates metrics. It
+ * registers with {@link MetricsSystem}, which periodically polls it to collect
+ * {@link MetricsRecord} and passes it to {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface MetricsSource {
   /**
-   * Get metrics from the source
+   * Get metrics from the metrics source
    * @param collector to contain the resulting metrics snapshot
    * @param all if true, return all metrics even if unchanged.
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java

@@ -22,7 +22,18 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * The metrics system interface
+ * The metrics system interface.
+ * 
+ * The following components are used for metrics.
+ * <ul>
+ * <li>{@link MetricsSource} generate and update metrics information.</li>
+ * <li>{@link MetricsSink} consume the metrics information</li>
+ * </ul>
+ * 
+ * {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ * system. Implementations of {@link MetricsSystem} polls the
+ * {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ * {@link MetricsSink}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/annotation/Metric.java

@@ -24,7 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * Annotation interface for a single metric
+ * Annotation interface for a single metric used to annotate a field or a method
+ * in the class.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordBuilderImpl.java

@@ -32,6 +32,14 @@ import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.lib.Interns;
 import org.apache.hadoop.util.Time;
 
+/**
+ * {@link MetricsRecordBuilder} implementation used for building metrics records
+ * by the {@link MetricsCollector}. It provides the following functionality:
+ * <ul>
+ * <li>Allows configuring filters for metrics.
+ * </ul>
+ *
+ */
 class MetricsRecordBuilderImpl extends MetricsRecordBuilder {
   private final MetricsCollector parent;
   private final long timestamp;
@@ -41,9 +49,15 @@ class MetricsRecordBuilderImpl extends MetricsRecordBuilder {
   private final MetricsFilter recordFilter, metricFilter;
   private final boolean acceptable;
 
+  /**
+   * @param parent {@link MetricsCollector} using this record builder
+   * @param info metrics information
+   * @param rf
+   * @param mf
+   * @param acceptable
+   */
   MetricsRecordBuilderImpl(MetricsCollector parent, MetricsInfo info,
-                           MetricsFilter rf, MetricsFilter mf,
-                           boolean acceptable) {
+      MetricsFilter rf, MetricsFilter mf, boolean acceptable) {
     this.parent = parent;
     timestamp = Time.now();
     recInfo = info;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java

@@ -37,7 +37,7 @@ class MetricsRecordImpl extends AbstractMetricsRecord {
 
   /**
    * Construct a metrics record
-   * @param info  {@link MetricInfo} of the record
+   * @param info  {@link MetricsInfo} of the record
    * @param timestamp of the record
    * @param tags  of the record
    * @param metrics of the record

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java

@@ -235,11 +235,9 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
   void registerSource(String name, String desc, MetricsSource source) {
     checkNotNull(config, "config");
     MetricsConfig conf = sourceConfigs.get(name);
-    MetricsSourceAdapter sa = conf != null
-        ? new MetricsSourceAdapter(prefix, name, desc, source,
-                                   injectedTags, period, conf)
-        : new MetricsSourceAdapter(prefix, name, desc, source,
-          injectedTags, period, config.subset(SOURCE_KEY));
+    MetricsSourceAdapter sa = new MetricsSourceAdapter(prefix, name, desc,
+        source, injectedTags, period, conf != null ? conf
+            : config.subset(SOURCE_KEY));
     sources.put(name, sa);
     sa.start();
     LOG.debug("Registered source "+ name);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java

@@ -27,8 +27,13 @@ import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
- * The default metrics system singleton
+ * The default metrics system singleton. This class is used by all the daemon
+ * processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ * initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ * to initialize the {@link MetricsSystem}.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
@@ -37,7 +42,10 @@ public enum DefaultMetricsSystem {
 
   private AtomicReference<MetricsSystem> impl =
       new AtomicReference<MetricsSystem>(new MetricsSystemImpl());
+  
+  @VisibleForTesting
   volatile boolean miniClusterMode = false;
+  
   final UniqueNames mBeanNames = new UniqueNames();
   final UniqueNames sourceNames = new UniqueNames();
 
@@ -87,12 +95,12 @@ public enum DefaultMetricsSystem {
 
   MetricsSystem getImpl() { return impl.get(); }
 
-  @InterfaceAudience.Private
+  @VisibleForTesting
   public static void setMiniClusterMode(boolean choice) {
     INSTANCE.miniClusterMode = choice;
   }
 
-  @InterfaceAudience.Private
+  @VisibleForTesting
   public static boolean inMiniClusterMode() {
     return INSTANCE.miniClusterMode;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java

@@ -35,7 +35,16 @@ import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 
 /**
- * Helper class to build metrics source object from annotations
+ * Helper class to build {@link MetricsSource} object from annotations.
+ * <p>
+ * For a given source object:
+ * <ul>
+ * <li>Sets the {@link Field}s annotated with {@link Metric} to
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * <li>
+ * For {@link Method}s annotated with {@link Metric} creates
+ * {@link MutableMetric} and adds it to the {@link MetricsRegistry}.</li>
+ * </ul>
  */
 @InterfaceAudience.Private
 public class MetricsSourceBuilder {
@@ -115,9 +124,15 @@ public class MetricsSourceBuilder {
     return r;
   }
 
+  /**
+   * Change the declared field {@code field} in {@code source} Object to
+   * {@link MutableMetric}
+   */
   private void add(Object source, Field field) {
     for (Annotation annotation : field.getAnnotations()) {
-      if (!(annotation instanceof Metric)) continue;
+      if (!(annotation instanceof Metric)) {
+        continue;
+      }
       try {
         // skip fields already set
         field.setAccessible(true);
@@ -131,7 +146,7 @@ public class MetricsSourceBuilder {
                                                   registry);
       if (mutable != null) {
         try {
-          field.set(source, mutable);
+          field.set(source, mutable); // Set the source field to MutableMetric
           hasAtMetric = true;
         } catch (Exception e) {
           throw new MetricsException("Error setting field "+ field +
@@ -141,9 +156,12 @@ public class MetricsSourceBuilder {
     }
   }
 
+  /** Add {@link MutableMetric} for a method annotated with {@link Metric} */
   private void add(Object source, Method method) {
     for (Annotation annotation : method.getAnnotations()) {
-      if (!(annotation instanceof Metric)) continue;
+      if (!(annotation instanceof Metric)) {
+        continue;
+      }
       factory.newForMethod(source, method, (Metric) annotation, registry);
       hasAtMetric = true;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java

@@ -130,6 +130,10 @@ public class MutableMetricsFactory {
     return Interns.info(name2, about.isEmpty() ? name2 : about);
   }
 
+  /**
+   * Remove the prefix "get", if any, from the method name. Return the
+   * capacitalized method name."
+   */
   protected String getName(Method method) {
     String methodName = method.getName();
     if (methodName.startsWith("get")) {
@@ -140,12 +144,15 @@ public class MutableMetricsFactory {
 
   protected MetricsInfo getInfo(Metric annotation, String defaultName) {
     String[] value = annotation.value();
-     if (value.length == 2) {
+    if (value.length == 2) {
+      // Use name and description from the annotation
       return Interns.info(value[0], value[1]);
     }
     if (value.length == 1) {
+      // Use description from the annotation and method name as metric name
       return Interns.info(defaultName, value[0]);
     }
+    // Use method name as metric name and description
     return Interns.info(defaultName, defaultName);
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.metrics2.lib;
 
 import static org.apache.hadoop.metrics2.lib.Interns.info;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -150,14 +149,8 @@ public class MutableQuantiles extends MutableMetric {
     @Override
     public void run() {
       synchronized (parent) {
-        try {
-          parent.previousCount = parent.estimator.getCount();
-          parent.previousSnapshot = parent.estimator.snapshot();
-        } catch (IOException e) {
-          // Couldn't get a new snapshot because the window was empty
-          parent.previousCount = 0;
-          parent.previousSnapshot = null;
-        }
+        parent.previousCount = parent.estimator.getCount();
+        parent.previousSnapshot = parent.estimator.snapshot();
         parent.estimator.clear();
       }
       parent.setChanged();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/Quantile.java

@@ -20,12 +20,14 @@ package org.apache.hadoop.metrics2.util;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 
+import com.google.common.collect.ComparisonChain;
+
 /**
  * Specifies a quantile (with error bounds) to be watched by a
  * {@link SampleQuantiles} object.
  */
 @InterfaceAudience.Private
-public class Quantile {
+public class Quantile implements Comparable<Quantile> {
   public final double quantile;
   public final double error;
 
@@ -57,4 +59,19 @@ public class Quantile {
     return (int) (Double.doubleToLongBits(quantile) ^ Double
         .doubleToLongBits(error));
   }
+
+  @Override
+  public int compareTo(Quantile other) {
+    return ComparisonChain.start()
+        .compare(quantile, other.quantile)
+        .compare(error, other.error)
+        .result();
+  }
+  
+  @Override
+  public String toString() {
+    return String.format("%.2f %%ile +/- %.2f%%",
+        quantile * 100, error * 100);
+  }
+
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java

@@ -18,16 +18,17 @@
 
 package org.apache.hadoop.metrics2.util;
 
-import java.io.IOException;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 
 /**
  * Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm
@@ -202,10 +203,8 @@ public class SampleQuantiles {
    * @param quantile Queried quantile, e.g. 0.50 or 0.99.
    * @return Estimated value at that quantile.
    */
-  private long query(double quantile) throws IOException {
-    if (samples.size() == 0) {
-      throw new IOException("No samples present");
-    }
+  private long query(double quantile) {
+    Preconditions.checkState(!samples.isEmpty(), "no data in estimator");
 
     int rankMin = 0;
     int desired = (int) (quantile * count);
@@ -231,14 +230,18 @@ public class SampleQuantiles {
   /**
    * Get a snapshot of the current values of all the tracked quantiles.
    * 
-   * @return snapshot of the tracked quantiles
-   * @throws IOException
-   *           if no items have been added to the estimator
+   * @return snapshot of the tracked quantiles. If no items are added
+   * to the estimator, returns null.
    */
-  synchronized public Map<Quantile, Long> snapshot() throws IOException {
+  synchronized public Map<Quantile, Long> snapshot() {
     // flush the buffer first for best results
     insertBatch();
-    Map<Quantile, Long> values = new HashMap<Quantile, Long>(quantiles.length);
+    
+    if (samples.isEmpty()) {
+      return null;
+    }
+    
+    Map<Quantile, Long> values = new TreeMap<Quantile, Long>();
     for (int i = 0; i < quantiles.length; i++) {
       values.put(quantiles[i], query(quantiles[i].quantile));
     }
@@ -273,6 +276,16 @@ public class SampleQuantiles {
     bufferCount = 0;
     samples.clear();
   }
+  
+  @Override
+  synchronized public String toString() {
+    Map<Quantile, Long> data = snapshot();
+    if (data == null) {
+      return "[no samples]";
+    } else {
+      return Joiner.on("\n").withKeyValueSeparator(": ").join(data);
+    }
+  }
 
   /**
    * Describes a measured value passed to the estimator, tracking additional

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMapping.java

@@ -48,7 +48,7 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
       throw new RuntimeException("Bailing out since native library couldn't " +
         "be loaded");
     }
-    LOG.info("Using JniBasedUnixGroupsMapping for Group resolution");
+    LOG.debug("Using JniBasedUnixGroupsMapping for Group resolution");
   }
 
   @Override
@@ -57,7 +57,11 @@ public class JniBasedUnixGroupsMapping implements GroupMappingServiceProvider {
     try {
       groups = getGroupForUser(user);
     } catch (Exception e) {
-      LOG.warn("Error getting groups for " + user, e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error getting groups for " + user, e);
+      } else {
+        LOG.info("Error getting groups for " + user + ": " + e.getMessage());
+      }
     }
     return Arrays.asList(groups);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.java

@@ -52,7 +52,7 @@ public class JniBasedUnixGroupsNetgroupMapping
       throw new RuntimeException("Bailing out since native library couldn't " +
         "be loaded");
     }
-    LOG.info("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
+    LOG.debug("Using JniBasedUnixGroupsNetgroupMapping for Netgroup resolution");
   }
 
   /**
@@ -115,7 +115,12 @@ public class JniBasedUnixGroupsNetgroupMapping
       // JNI code does not expect '@' at the begining of the group name
       users = getUsersForNetgroupJNI(netgroup.substring(1));
     } catch (Exception e) {
-      LOG.warn("error getting users for netgroup " + netgroup, e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error getting users for netgroup " + netgroup, e);
+      } else {
+        LOG.info("Error getting users for netgroup " + netgroup + 
+            ": " + e.getMessage());
+      }
     }
     if (users != null && users.length != 0) {
       return Arrays.asList(users);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java

@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Checksum {
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Checksum {
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
 
   /*
    * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
    * See also TestPureJavaCrc32.Table.
    */
-  private static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+  	/* T8_0 */
     0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 
     0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 
     0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 
@@ -154,9 +180,8 @@ public class PureJavaCrc32 implements Checksum {
     0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 
     0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 
     0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 
-    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
-  };
-  private static final int[] T8_1 = new int[] {
+    0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+    /* T8_1 */
     0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, 
     0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, 
     0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, 
@@ -220,9 +245,8 @@ public class PureJavaCrc32 implements Checksum {
     0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, 
     0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, 
     0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, 
-    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
-  };
-  private static final int[] T8_2 = new int[] {
+    0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+    /* T8_2 */
     0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, 
     0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, 
     0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, 
@@ -286,9 +310,8 @@ public class PureJavaCrc32 implements Checksum {
     0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, 
     0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, 
     0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, 
-    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
-  };
-  private static final int[] T8_3 = new int[] {
+    0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+    /* T8_3 */
     0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, 
     0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, 
     0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, 
@@ -352,9 +375,8 @@ public class PureJavaCrc32 implements Checksum {
     0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, 
     0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, 
     0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, 
-    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
-  };
-  private static final int[] T8_4 = new int[] {
+    0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+    /* T8_4 */
     0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, 
     0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, 
     0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, 
@@ -418,9 +440,8 @@ public class PureJavaCrc32 implements Checksum {
     0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, 
     0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, 
     0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, 
-    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
-  };
-  private static final int[] T8_5 = new int[] {
+    0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+    /* T8_5 */
     0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, 
     0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, 
     0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, 
@@ -484,9 +505,8 @@ public class PureJavaCrc32 implements Checksum {
     0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, 
     0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, 
     0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, 
-    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
-  };
-  private static final int[] T8_6 = new int[] {
+    0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+    /* T8_6 */
     0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, 
     0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, 
     0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, 
@@ -550,9 +570,8 @@ public class PureJavaCrc32 implements Checksum {
     0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, 
     0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, 
     0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, 
-    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
-  };
-  private static final int[] T8_7 = new int[] {
+    0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+    /* T8_7 */
     0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, 
     0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, 
     0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java

@@ -56,22 +56,38 @@ public class PureJavaCrc32C implements Checksum {
   @Override
   public void update(byte[] b, int off, int len) {
     int localCrc = crc;
+
     while(len > 7) {
-      int c0 = b[off++] ^ localCrc;
-      int c1 = b[off++] ^ (localCrc >>>= 8);
-      int c2 = b[off++] ^ (localCrc >>>= 8);
-      int c3 = b[off++] ^ (localCrc >>>= 8);
-      localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
-          ^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
+      final int c0 =(b[off+0] ^ localCrc) & 0xff;
+      final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
+      final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
+      final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
+      localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+          ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+      final int c4 = b[off+4] & 0xff;
+      final int c5 = b[off+5] & 0xff;
+      final int c6 = b[off+6] & 0xff;
+      final int c7 = b[off+7] & 0xff;
 
-      localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
-           ^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
+      localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+           ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
 
+      off += 8;
       len -= 8;
     }
-    while(len > 0) {
-      localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
-      len--;
+
+    /* loop unroll - duff's device style */
+    switch(len) {
+      case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
+      default:
+        /* nothing */
     }
     
     // Publish crc out to object
@@ -80,14 +96,24 @@ public class PureJavaCrc32C implements Checksum {
 
   @Override
   final public void update(int b) {
-    crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
+    crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
   }
     
   // CRC polynomial tables generated by:
   // java -cp build/test/classes/:build/classes/ \
   //   org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
 
-  static final int[] T8_0 = new int[] {
+  private static final int T8_0_start = 0*256;
+  private static final int T8_1_start = 1*256;
+  private static final int T8_2_start = 2*256;
+  private static final int T8_3_start = 3*256;
+  private static final int T8_4_start = 4*256;
+  private static final int T8_5_start = 5*256;
+  private static final int T8_6_start = 6*256;
+  private static final int T8_7_start = 7*256;
+
+  private static final int[] T = new int[] {
+    /* T8_0 */
     0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 
     0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 
     0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 
@@ -151,9 +177,8 @@ public class PureJavaCrc32C implements Checksum {
     0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 
     0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 
     0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 
-    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
-  };
-  static final int[] T8_1 = new int[] {
+    0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, 
+    /* T8_1 */
     0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 
     0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, 
     0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 
@@ -217,9 +242,8 @@ public class PureJavaCrc32C implements Checksum {
     0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 
     0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 
     0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, 
-    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
-  };
-  static final int[] T8_2 = new int[] {
+    0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, 
+    /* T8_2 */
     0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 
     0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, 
     0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 
@@ -283,9 +307,8 @@ public class PureJavaCrc32C implements Checksum {
     0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 
     0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, 
     0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, 
-    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
-  };
-  static final int[] T8_3 = new int[] {
+    0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, 
+    /* T8_3 */
     0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 
     0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, 
     0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 
@@ -349,9 +372,8 @@ public class PureJavaCrc32C implements Checksum {
     0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 
     0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, 
     0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, 
-    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
-  };
-  static final int[] T8_4 = new int[] {
+    0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, 
+    /* T8_4 */
     0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 
     0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, 
     0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 
@@ -415,9 +437,8 @@ public class PureJavaCrc32C implements Checksum {
     0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 
     0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 
     0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, 
-    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
-  };
-  static final int[] T8_5 = new int[] {
+    0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, 
+    /* T8_5 */
     0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 
     0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, 
     0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 
@@ -481,9 +502,8 @@ public class PureJavaCrc32C implements Checksum {
     0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 
     0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 
     0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, 
-    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
-  };
-  static final int[] T8_6 = new int[] {
+    0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, 
+    /* T8_6 */
     0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 
     0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, 
     0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 
@@ -547,9 +567,8 @@ public class PureJavaCrc32C implements Checksum {
     0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 
     0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 
     0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, 
-    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
-  };
-  static final int[] T8_7 = new int[] {
+    0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, 
+    /* T8_7 */
     0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 
     0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, 
     0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 
@@ -613,6 +632,6 @@ public class PureJavaCrc32C implements Checksum {
     0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 
     0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, 
     0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 
-    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
+    0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 
   };
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -608,7 +608,8 @@ public class StringUtils {
             "  build = " + VersionInfo.getUrl() + " -r "
                          + VersionInfo.getRevision()  
                          + "; compiled by '" + VersionInfo.getUser()
-                         + "' on " + VersionInfo.getDate()}
+                         + "' on " + VersionInfo.getDate(),
+            "  java = " + System.getProperty("java.version") }
         )
       );
 

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/conffile

@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/etc/hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/control

@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Package: hadoop-common
-Version: @version@
-Section: misc
-Priority: optional
-Provides: hadoop-common
-Architecture: all
-Depends: openjdk-6-jre-headless
-Maintainer: Apache Software Foundation <general@hadoop.apache.org>
-Description: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing.
-Distribution: development

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/postinst

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hadoop-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/postrm

@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-/usr/sbin/groupdel hadoop 2> /dev/null >dev/null
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/preinst

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop

hadoop-common-project/hadoop-common/src/main/packages/deb/hadoop.control/prerm

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hadoop-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop \
-  --uninstal

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-datanode

@@ -1,151 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-datanode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-datanode: start and stop the Apache Hadoop Data Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-  DN_USER="root"
-  IDENT_USER=${HADOOP_SECURE_DN_USER}
-else
-  DN_USER="hdfs"
-  IDENT_USER=${DN_USER}
-fi
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-datanode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-datanode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Data Node server not in use (/etc/hadoop/hadoop-datanode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-datanode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-jobtracker

@@ -1,143 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-jobtracker	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Job Tracker server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-jobtracker: start and stop the Apache Hadoop Job Tracker daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-jobtracker_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-jobtracker_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Job Tracker server not in use (/etc/hadoop/hadoop-jobtracker_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid ${JAVA_HOME}/bin/java hadoop-jobtracker && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-jobtracker {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-namenode

@@ -1,155 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-namenode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-namenode: start and stop the Apache Hadoop Name Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-namenode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-namenode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Name Node server not in use (/etc/hadoop/hadoop-namenode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-format() {
-    sudo -u hdfs ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  format)
-	log_daemon_msg "Formatting Apache Hadoop Name Node" "hadoop-namenode"
-	format
-	if [ $? -eq 0 ]; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid ${JAVA_HOME}/bin/java hadoop-namenode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-namenode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-tasktracker

@@ -1,143 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-tasktracker	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Task Tracker server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-tasktracker: start and stop the Apache Hadoop Task Tracker daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-tasktracker_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-tasktracker_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Task Tracker server not in use (/etc/hadoop/hadoop-tasktracker_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid ${JAVA_HOME}/bin/java hadoop-tasktracker && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-tasktracker {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/hadoop-create-user.sh

@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-this="$bin/$script"
-
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Require parameter:
-     --config /etc/hadoop                                  Location of Hadoop configuration file
-     -u <username>                                         Create user on HDFS
-  Optional parameters:
-     -h                                                    Display this message
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                 Set Kerberos realm
-     --super-user=hdfs                                     Set super user id
-     --super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'kerberos-realm:' \
-  -l 'super-user:' \
-  -l 'super-user-keytab:' \
-  -o 'h' \
-  -o 'u' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-    exit 1
-fi
-
-create_user() {
-  if [ "${SETUP_USER}" = "" ]; then
-    break
-  fi
-  HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-  export HADOOP_PREFIX
-  export HADOOP_CONF_DIR
-  export JAVA_HOME
-  export SETUP_USER=${SETUP_USER}
-  export SETUP_PATH=/user/${SETUP_USER}
-
-  if [ ! "${KERBEROS_REALM}" = "" ]; then
-    # locate kinit cmd
-    if [ -e /etc/lsb-release ]; then
-      KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    else
-      KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    fi
-    su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER} 
-  fi
-
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 711 ${SETUP_PATH}" ${HADOOP_HDFS_USER}
-
-  if [ "$?" == "0" ]; then
-    echo "User directory has been setup: ${SETUP_PATH}"
-  fi
-}
-
-eval set -- "${OPTS}"
-while true; do
-  case "$1" in
-    -u)
-      shift
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      ;;
-    --super-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      ;;
-    --super-user-keytab)
-      HDFS_USER_KEYTAB=$2; shift 2
-      ;;
-    -h)
-      usage
-      ;; 
-    --)
-      while shift; do
-        SETUP_USER=$1
-        create_user
-      done
-      break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done 
-

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-applications.sh

@@ -1,142 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-this="$bin/$script"
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Require parameter:
-     --config /etc/hadoop                                  Location of Hadoop configuration file
-     --apps=<csl of apps:user hcat:hcat,hbase,hive:user>   Apps you want to setup on hdfs
-                                                           If user is not specified, app name
-                                                           will be used as the user name as well
-  Optional parameters:
-     -h                                                    Display this message
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                 Set Kerberos realm
-     --super-user=hdfs                                     Set super user id
-     --super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'kerberos-realm:' \
-  -l 'super-user:' \
-  -l 'super-user-keytab:' \
-  -l 'apps:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-    exit 1
-fi
-
-function setup_apps
-{
-  if [ -z $APPS ] 
-  then
-    usage
-    break
-  fi
-
-  #if super user is not set default to hdfs
-  HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-
-  if [ ! "${KERBEROS_REALM}" = "" ]; then
-    # locate kinit cmd
-    if [ -e /etc/lsb-release ]; then
-      KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    else
-      KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
-    fi
-    su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER}
-  fi
-  #process each app
-  oldIFS=$IFS 
-  IFS=','
-  for app in $APPS
-  do
-    IFS=":"
-    arr=($app)
-    app=${arr[0]}
-    user=${arr[1]}
-    IFS=','
-    #if user is empty, default it to app
-    if [ -z $user ]
-    then
-      user=$app
-    fi
-    
-    path="/apps/${app}"
-
-    #create the dir
-    cmd="su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir ${path}' ${HADOOP_HDFS_USER}"
-    echo $cmd
-    eval $cmd
-
-    #make owner to be the app
-    cmd="su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${user} ${path}' ${HADOOP_HDFS_USER}"
-    echo $cmd
-    eval $cmd
-
-    if [ "$?" == "0" ]; then
-      echo "App directory has been setup: ${path}"
-    fi
-  done
-  IFS=$oldIFS
-}
-
-eval set -- "${OPTS}"
-while true; do
-  case "$1" in
-    --apps)
-      APPS=$2; shift 2
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      ;;
-    --super-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      ;;
-    --super-user-keytab)
-      HDFS_USER_KEYTAB=$2; shift 2
-      ;;
-    -h)
-      usage
-      ;; 
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done
-
-setup_apps

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh

@@ -1,707 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-this="${BASH_SOURCE-$0}"
-HADOOP_DEFAULT_PREFIX=`dirname "$this"`/..
-HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}
-export HADOOP_PREFIX
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --auto                                                          Setup path and configuration automatically
-     --default                                                       Setup configuration as default
-     --conf-dir=/etc/hadoop                                          Set configuration directory
-     --datanode-dir=/var/lib/hadoop/hdfs/datanode                    Set datanode directory
-     --group=hadoop                                                  Set Hadoop group name
-     -h                                                              Display this message
-     --hdfs-user=hdfs                                                Set HDFS user
-     --jobtracker-host=hostname                                      Set jobtracker host
-     --namenode-host=hostname                                        Set namenode host
-     --secondarynamenode-host=hostname                               Set secondary namenode host
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-     --kinit-location=/usr/kerberos/bin/kinit                        Set kinit location
-     --keytab-dir=/etc/security/keytabs                              Set keytab directory
-     --log-dir=/var/log/hadoop                                       Set log directory
-     --pid-dir=/var/run/hadoop                                       Set pid directory
-     --hdfs-dir=/var/lib/hadoop/hdfs                                 Set HDFS directory
-     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
-     --mapred-dir=/var/lib/hadoop/mapred                             Set mapreduce directory
-     --mapreduce-user=mr                                             Set mapreduce user
-     --mapreduce-user-keytab=/home/mr/hdfs.keytab                    Set mapreduce user key tab
-     --namenode-dir=/var/lib/hadoop/hdfs/namenode                    Set namenode directory
-     --replication=3                                                 Set replication factor
-     --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler  Set task scheduler
-     --datanodes=hostname1,hostname2,...                             SET the datanodes
-     --tasktrackers=hostname1,hostname2,...                          SET the tasktrackers
-     --dfs-webhdfs-enabled=false|true                                Enable webhdfs
-     --dfs-support-append=false|true                                 Enable append
-     --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts'    Setup proxy users for hadoop
-     --hbase-user=hbase                                              User which hbase is running as. Defaults to hbase
-     --mapreduce-cluster-mapmemory-mb=memory                         Virtual memory of a map slot for the MR framework. Defaults to -1
-     --mapreduce-cluster-reducememory-mb=memory                      Virtual memory, of a reduce slot for the MR framework. Defaults to -1
-     --mapreduce-jobtracker-maxmapmemory-mb=memory                   Maximum virtual memory of a single map task. Defaults to -1
-                                                                     This value should be set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
-     --mapreduce-jobtracker-maxreducememory-mb=memory                Maximum virtual memory of a single reduce task. Defaults to -1
-                                                                     This value should be set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
-     --mapreduce-map-memory-mb=memory                                Virtual memory of a single map slot for a job. Defaults to -1
-                                                                     This value should be <= mapred.cluster.max.map.memory.mb
-     --mapreduce-reduce-memory-mb=memory                             Virtual memory, of a single reduce slot for a job. Defaults to -1
-                                                                     This value should be <= mapred.cluster.max.reduce.memory.mb
-     --dfs-datanode-dir-perm=700                                     Set the permission for the datanode data directories. Defaults to 700
-     --dfs-block-local-path-access-user=user                         User for which you want to enable shortcircuit read.
-     --dfs-client-read-shortcircuit=true/false                       Enable shortcircuit read for the client. Will default to true if the shortcircuit user is set.
-     --dfs-client-read-shortcircuit-skip-checksum=false/true         Disable checking of checksum when shortcircuit read is taking place. Defaults to false.
-  "
-  exit 1
-}
-
-check_permission() {
-  TARGET=$1
-  OWNER="0"
-  RESULT=0
-  while [ "$TARGET" != "/" ]; do
-    if [ "`uname`" = "Darwin" ]; then
-      OWNER=`stat -f %u $TARGET`
-    else
-      OWNER=`stat -c %u $TARGET`
-    fi
-    if [ "$OWNER" != "0" ]; then
-      RESULT=1
-      break
-    fi
-    TARGET=`dirname $TARGET`
-  done
-  return $RESULT
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  if [ -e $2 ]; then
-    mv -f $2 "$2.bak"
-  fi
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-#########################################
-# Function to modify a value of a field in an xml file
-# Params: $1 is the file with full path; $2 is the property, $3 is the new value
-#########################################
-function addPropertyToXMLConf
-{
-  #read the file name with full path
-  local file=$1
-  #get the property name
-  local property=$2
-  #get what value should be set for that
-  local propValue=$3
-  #get the description
-  local desc=$4
-  #get the value for the final tag
-  local finalVal=$5
-
-  #create the property text, make sure the / are escaped
-  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>\n"
-  #if description is not empty add it
-  if [ ! -z $desc ]
-  then
-    propText="${propText}<description>$desc<\/description>\n"
-  fi
-  
-  #if final is not empty add it
-  if [ ! -z $finalVal ]
-  then
-    propText="${propText}final>$finalVal<\/final>\n"
-  fi
-
-  #add the ending tag
-  propText="${propText}<\/property>\n"
-
-  #add the property to the file
-  endText="<\/configuration>"
-  #add the text using sed at the end of the file
-  sed -i "s|$endText|$propText$endText|" $file
-}
-
-##########################################
-# Function to setup up the short circuit read settings
-#########################################
-function setupShortCircuitRead
-{
-  local conf_file="${HADOOP_CONF_DIR}/hdfs-site.xml"
-  #if the shortcircuit user is not set then return
-  if [ -z $DFS_BLOCK_LOCAL_PATH_ACCESS_USER ]
-  then
-    return
-  fi
-  
-  #set the defaults if values not present
-  DFS_CLIENT_READ_SHORTCIRCUIT=${DFS_CLIENT_READ_SHORTCIRCUIT:-false}
-  DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=${DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM:-false}
-
-  #add the user to the conf file
-  addPropertyToXMLConf "$conf_file" "dfs.block.local-path-access.user" "$DFS_BLOCK_LOCAL_PATH_ACCESS_USER"
-  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit" "$DFS_CLIENT_READ_SHORTCIRCUIT"
-  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit.skip.checksum" "$DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM"
-}
-
-##########################################
-# Function to setup up the proxy user settings
-#########################################
-function setupProxyUsers
-{
-  local conf_file="${HADOOP_CONF_DIR}/core-site.xml"
-  #if hadoop proxy users are sent, setup hadoop proxy
-  if [ ! -z $HADOOP_PROXY_USERS ]
-  then
-    oldIFS=$IFS
-    IFS=';'
-    #process each proxy config
-    for proxy in $HADOOP_PROXY_USERS
-    do
-      #get the user, group and hosts information for each proxy
-      IFS=':'
-      arr=($proxy)
-      user="${arr[0]}"
-      groups="${arr[1]}"
-      hosts="${arr[2]}"
-      #determine the property names and values
-      proxy_groups_property="hadoop.proxyuser.${user}.groups"
-      proxy_groups_val="$groups"
-      addPropertyToXMLConf "$conf_file" "$proxy_groups_property" "$proxy_groups_val"
-      proxy_hosts_property="hadoop.proxyuser.${user}.hosts"
-      proxy_hosts_val="$hosts"
-      addPropertyToXMLConf "$conf_file" "$proxy_hosts_property" "$proxy_hosts_val"
-      IFS=';'
-    done
-    IFS=$oldIFS
-  fi
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'auto' \
-  -l 'java-home:' \
-  -l 'conf-dir:' \
-  -l 'default' \
-  -l 'group:' \
-  -l 'hdfs-dir:' \
-  -l 'namenode-dir:' \
-  -l 'datanode-dir:' \
-  -l 'mapred-dir:' \
-  -l 'namenode-host:' \
-  -l 'secondarynamenode-host:' \
-  -l 'jobtracker-host:' \
-  -l 'log-dir:' \
-  -l 'pid-dir:' \
-  -l 'replication:' \
-  -l 'taskscheduler:' \
-  -l 'hdfs-user:' \
-  -l 'hdfs-user-keytab:' \
-  -l 'mapreduce-user:' \
-  -l 'mapreduce-user-keytab:' \
-  -l 'keytab-dir:' \
-  -l 'kerberos-realm:' \
-  -l 'kinit-location:' \
-  -l 'datanodes:' \
-  -l 'tasktrackers:' \
-  -l 'dfs-webhdfs-enabled:' \
-  -l 'hadoop-proxy-users:' \
-  -l 'dfs-support-append:' \
-  -l 'hbase-user:' \
-  -l 'mapreduce-cluster-mapmemory-mb:' \
-  -l 'mapreduce-cluster-reducememory-mb:' \
-  -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
-  -l 'mapreduce-jobtracker-maxreducememory-mb:' \
-  -l 'mapreduce-map-memory-mb:' \
-  -l 'mapreduce-reduce-memory-mb:' \
-  -l 'dfs-datanode-dir-perm:' \
-  -l 'dfs-block-local-path-access-user:' \
-  -l 'dfs-client-read-shortcircuit:' \
-  -l 'dfs-client-read-shortcircuit-skip-checksum:' \
-  -o 'h' \
-  -- "$@") 
-  
-if [ $? != 0 ] ; then
-    usage
-fi
-
-# Make sure the HADOOP_LOG_DIR is not picked up from user environment.
-unset HADOOP_LOG_DIR
-  
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --auto)
-      AUTOSETUP=1
-      AUTOMATED=1
-      shift
-      ;; 
-    --java-home)
-      JAVA_HOME=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --conf-dir)
-      HADOOP_CONF_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --default)
-      AUTOMATED=1; shift
-      ;;
-    --group)
-      HADOOP_GROUP=$2; shift 2
-      AUTOMATED=1
-      ;;
-    -h)
-      usage
-      ;; 
-    --hdfs-dir)
-      HADOOP_HDFS_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --namenode-dir)
-      HADOOP_NN_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --datanode-dir)
-      HADOOP_DN_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --mapred-dir)
-      HADOOP_MAPRED_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --namenode-host)
-      HADOOP_NN_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --secondarynamenode-host)
-      HADOOP_SNN_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --jobtracker-host)
-      HADOOP_JT_HOST=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --log-dir)
-      HADOOP_LOG_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --pid-dir)
-      HADOOP_PID_DIR=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --replication)
-      HADOOP_REPLICATION=$2; shift 2
-      AUTOMATED=1
-      ;; 
-    --taskscheduler)
-      HADOOP_TASK_SCHEDULER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user)
-      HADOOP_MR_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --keytab-dir)
-      KEYTAB_DIR=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user-keytab)
-      HDFS_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user-keytab)
-      MR_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      SECURITY_TYPE="kerberos"
-      AUTOMATED=1
-      ;;
-    --kinit-location)
-      KINIT=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --datanodes)
-      DATANODES=$2; shift 2
-      AUTOMATED=1
-      DATANODES=$(echo $DATANODES | tr ',' ' ')
-      ;;
-    --tasktrackers)
-      TASKTRACKERS=$2; shift 2
-      AUTOMATED=1
-      TASKTRACKERS=$(echo $TASKTRACKERS | tr ',' ' ')
-      ;;
-    --dfs-webhdfs-enabled)
-      DFS_WEBHDFS_ENABLED=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hadoop-proxy-users)
-      HADOOP_PROXY_USERS=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-support-append)
-      DFS_SUPPORT_APPEND=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hbase-user)
-      HBASE_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-cluster-mapmemory-mb)
-      MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-cluster-reducememory-mb)
-      MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-jobtracker-maxmapmemory-mb)
-      MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-jobtracker-maxreducememory-mb)
-      MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-map-memory-mb)
-      MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-reduce-memory-mb)
-      MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-datanode-dir-perm)
-      DFS_DATANODE_DIR_PERM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-block-local-path-access-user)
-      DFS_BLOCK_LOCAL_PATH_ACCESS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-client-read-shortcircuit)
-      DFS_CLIENT_READ_SHORTCIRCUIT=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --dfs-client-read-shortcircuit-skip-checksum)
-      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1 
-      ;;
-  esac
-done 
-
-AUTOSETUP=${AUTOSETUP:-1}
-JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
-HADOOP_NN_HOST=${HADOOP_NN_HOST:-`hostname`}
-HADOOP_SNN_HOST=${HADOOP_SNN_HOST:-`hostname`}
-HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
-HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`}
-HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs}
-HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
-HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop}
-HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/log/hadoop}
-HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
-HADOOP_REPLICATION=${HADOOP_RELICATION:-3}
-HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
-HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
-DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
-DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
-HBASE_USER=${HBASE_USER:-hbase}
-MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
-MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
-MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
-MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
-MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
-MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
-KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
-HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
-MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
-DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
-DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
-KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
-SECURITY_TYPE=${SECURITY_TYPE:-simple}
-KINIT=${KINIT:-/usr/kerberos/bin/kinit}
-#deault the data dir perm to 700
-DFS_DATANODE_DIR_PERM=${DFS_DATANODE_DIR_PERM:-700}
-if [ "${SECURITY_TYPE}" = "kerberos" ]; then
-  TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
-  HADOOP_DN_ADDR="0.0.0.0:1019"
-  HADOOP_DN_HTTP_ADDR="0.0.0.0:1022"
-  SECURITY="true"
-  HADOOP_SECURE_DN_USER=${HADOOP_HDFS_USER}
-else
-  TASK_CONTROLLER="org.apache.hadoop.mapred.DefaultTaskController"
-  HADOOP_DN_ADDR="0.0.0.0:50010"
-  HADOOP_DN_HTTP_ADDR="0.0.0.0:50075"
-  SECURITY="false"
-  HADOOP_SECURE_DN_USER=""
-fi
-
-#unset env vars
-unset HADOOP_CLIENT_OPTS HADOOP_NAMENODE_OPTS HADOOP_DATANODE_OPTS HADOOP_SECONDARYNAMENODE_OPTS HADOOP_JAVA_PLATFORM_OPTS
-
-if [ "${AUTOMATED}" != "1" ]; then
-  echo "Setup Hadoop Configuration"
-  echo
-  echo -n "Where would you like to put config directory? (${HADOOP_CONF_DIR}) "
-  read USER_HADOOP_CONF_DIR
-  echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) "
-  read USER_HADOOP_LOG_DIR
-  echo -n "Where would you like to put pid directory? (${HADOOP_PID_DIR}) "
-  read USER_HADOOP_PID_DIR
-  echo -n "What is the host of the namenode? (${HADOOP_NN_HOST}) "
-  read USER_HADOOP_NN_HOST
-  echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) "
-  read USER_HADOOP_NN_DIR
-  echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) "
-  read USER_HADOOP_DN_DIR
-  echo -n "What is the host of the jobtracker? (${HADOOP_JT_HOST}) "
-  read USER_HADOOP_JT_HOST
-  echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) "
-  read USER_HADOOP_MAPRED_DIR
-  echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) "
-  read USER_JAVA_HOME
-  echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) "
-  read USER_AUTOSETUP
-  echo
-  JAVA_HOME=${USER_USER_JAVA_HOME:-$JAVA_HOME}
-  HADOOP_NN_HOST=${USER_HADOOP_NN_HOST:-$HADOOP_NN_HOST}
-  HADOOP_NN_DIR=${USER_HADOOP_NN_DIR:-$HADOOP_NN_DIR}
-  HADOOP_DN_DIR=${USER_HADOOP_DN_DIR:-$HADOOP_DN_DIR}
-  HADOOP_JT_HOST=${USER_HADOOP_JT_HOST:-$HADOOP_JT_HOST}
-  HADOOP_HDFS_DIR=${USER_HADOOP_HDFS_DIR:-$HADOOP_HDFS_DIR}
-  HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR}
-  HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
-  HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR}
-  HADOOP_PID_DIR=${USER_HADOOP_PID_DIR:-$HADOOP_PID_DIR}
-  HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR}
-  AUTOSETUP=${USER_AUTOSETUP:-y}
-  echo "Review your choices:"
-  echo
-  echo "Config directory            : ${HADOOP_CONF_DIR}"
-  echo "Log directory               : ${HADOOP_LOG_DIR}"
-  echo "PID directory               : ${HADOOP_PID_DIR}"
-  echo "Namenode host               : ${HADOOP_NN_HOST}"
-  echo "Namenode directory          : ${HADOOP_NN_DIR}"
-  echo "Datanode directory          : ${HADOOP_DN_DIR}"
-  echo "Jobtracker host             : ${HADOOP_JT_HOST}"
-  echo "Mapreduce directory         : ${HADOOP_MAPRED_DIR}"
-  echo "Task scheduler              : ${HADOOP_TASK_SCHEDULER}"
-  echo "JAVA_HOME directory         : ${JAVA_HOME}"
-  echo "Create dirs/copy conf files : ${AUTOSETUP}"
-  echo
-  echo -n "Proceed with generate configuration? (y/N) "
-  read CONFIRM
-  if [ "${CONFIRM}" != "y" ]; then
-    echo "User aborted setup, exiting..."
-    exit 1
-  fi
-fi
-
-if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
-  if [ -d ${KEYTAB_DIR} ]; then
-    chmod 700 ${KEYTAB_DIR}/*
-    chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[jt]t.service.keytab
-    chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[dns]n.service.keytab
-  fi
-  chmod 755 -R ${HADOOP_PREFIX}/sbin/*hadoop*
-  chmod 755 -R ${HADOOP_PREFIX}/bin/hadoop
-
-  HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-${HADOOP_PREFIX}/libexec}
-  chmod 755 -R ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
-
-  mkdir -p /home/${HADOOP_MR_USER}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} /home/${HADOOP_MR_USER}
-  HDFS_DIR=`echo ${HADOOP_HDFS_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${HDFS_DIR}
-  if [ -e ${HADOOP_NN_DIR} ]; then
-    rm -rf ${HADOOP_NN_DIR}
-  fi
-  DATANODE_DIR=`echo ${HADOOP_DN_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${DATANODE_DIR}
-  MAPRED_DIR=`echo ${HADOOP_MAPRED_DIR} | sed -e 's/,/ /g'`
-  mkdir -p ${MAPRED_DIR}
-  mkdir -p ${HADOOP_CONF_DIR}
-  check_permission ${HADOOP_CONF_DIR}
-  if [ $? == 1 ]; then
-    echo "Full path to ${HADOOP_CONF_DIR} should be owned by root."
-    exit 1
-  fi
-
-  mkdir -p ${HADOOP_LOG_DIR}
-  #create the log sub dir for diff users
-  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-
-  mkdir -p ${HADOOP_PID_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HDFS_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${DATANODE_DIR}
-  chmod 700 -R ${DATANODE_DIR}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${MAPRED_DIR}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_PID_DIR}
-  chown root:${HADOOP_GROUP} ${HADOOP_PID_DIR}
-
-  #change the permission and the owner
-  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
-
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-
-  #setup up the proxy users
-  setupProxyUsers
- 
-  #setup short circuit read
-  setupShortCircuitRead
-
-  #set the owner of the hadoop dir to root
-  chown root ${HADOOP_PREFIX}
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
-  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
-  
-  #set taskcontroller
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
-  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
-
-
-  #generate the slaves file and include and exclude files for hdfs and mapred
-  echo '' > ${HADOOP_CONF_DIR}/slaves
-  echo '' > ${HADOOP_CONF_DIR}/dfs.include
-  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
-  echo '' > ${HADOOP_CONF_DIR}/mapred.include
-  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
-  for dn in $DATANODES
-  do
-    echo $dn >> ${HADOOP_CONF_DIR}/slaves
-    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
-  done
-  for tt in $TASKTRACKERS
-  do
-    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
-  done
-
-  echo "Configuration setup is completed."
-  if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then
-    echo "Proceed to run hadoop-setup-hdfs.sh on namenode."
-  fi
-else
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties
-  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
-  
-  #setup up the proxy users
-  setupProxyUsers
-  
-  #setup short circuit read
-  setupShortCircuitRead
-
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
-  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
-  #set taskcontroller
-  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
-  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
-  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
-  
-  #generate the slaves file and include and exclude files for hdfs and mapred
-  echo '' > ${HADOOP_CONF_DIR}/slaves
-  echo '' > ${HADOOP_CONF_DIR}/dfs.include
-  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
-  echo '' > ${HADOOP_CONF_DIR}/mapred.include
-  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
-  for dn in $DATANODES
-  do
-    echo $dn >> ${HADOOP_CONF_DIR}/slaves
-    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
-  done
-  for tt in $TASKTRACKERS
-  do
-    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
-  done
-  
-  echo
-  echo "Configuration file has been generated in:"
-  echo
-  echo "${HADOOP_CONF_DIR}/core-site.xml"
-  echo "${HADOOP_CONF_DIR}/hdfs-site.xml"
-  echo "${HADOOP_CONF_DIR}/mapred-site.xml"
-  echo "${HADOOP_CONF_DIR}/hadoop-env.sh"
-  echo "${HADOOP_CONF_DIR}/hadoop-policy.xml"
-  echo "${HADOOP_CONF_DIR}/commons-logging.properties"
-  echo "${HADOOP_CONF_DIR}/taskcontroller.cfg"
-  echo "${HADOOP_CONF_DIR}/capacity-scheduler.xml"
-  echo "${HADOOP_CONF_DIR}/log4j.properties"
-  echo "${HADOOP_CONF_DIR}/hadoop-metrics2.properties"
-  echo
-  echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode."
-fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-hdfs.sh

@@ -1,157 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --format                                                        Force namenode format
-     --group=hadoop                                                  Set Hadoop group
-     -h                                                              Display this message
-     --hdfs-user=hdfs                                                Set HDFS user
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
-     --mapreduce-user=mr                                             Set mapreduce user
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'format' \
-  -l 'hdfs-user:' \
-  -l 'hdfs-user-keytab:' \
-  -l 'mapreduce-user:' \
-  -l 'kerberos-realm:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --format)
-      FORMAT_NAMENODE=1; shift
-      AUTOMATED=1
-      ;;
-    --group)
-      HADOOP_GROUP=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user)
-      HADOOP_HDFS_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --mapreduce-user)
-      HADOOP_MR_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --yarn-user)
-      HADOOP_YARN_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --hdfs-user-keytab)
-      HDFS_KEYTAB=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
-HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
-HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-HADOOP_MAPREDUCE_USER=${HADOOP_MR_USER:-mapred}
-
-if [ "${KERBEROS_REALM}" != "" ]; then
-  # Determine kerberos location base on Linux distro.
-  if [ -e /etc/lsb-release ]; then
-    KERBEROS_BIN=/usr/bin
-  else
-    KERBEROS_BIN=/usr/kerberos/bin
-  fi
-  kinit_cmd="${KERBEROS_BIN}/kinit -k -t ${HDFS_KEYTAB} ${HADOOP_HDFS_USER}"
-  su -c "${kinit_cmd}" ${HADOOP_HDFS_USER}
-fi
-
-echo "Setup Hadoop Distributed File System"
-echo
-
-# Format namenode
-if [ "${FORMAT_NAMENODE}" == "1" ]; then
-  echo "Formatting namenode"
-  echo
-  su -c "echo Y | ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format" ${HADOOP_HDFS_USER}
-  echo
-fi
-
-# Start namenode process
-echo "Starting namenode process"
-echo
-if [ -e ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh ]; then
-  DAEMON_PATH=${HADOOP_PREFIX}/sbin
-else
-  DAEMON_PATH=${HADOOP_PREFIX}/bin
-fi
-su -c "${DAEMON_PATH}/hadoop-daemon.sh --config ${HADOOP_CONF_DIR} start namenode" ${HADOOP_HDFS_USER}
-echo
-echo "Initialize HDFS file system: "
-echo
-
-#create the /user dir 
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user" ${HADOOP_HDFS_USER}
-
-#create /tmp and give it 777
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp" ${HADOOP_HDFS_USER}
-
-#create /mapred
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /mapred" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 700 /mapred" ${HADOOP_HDFS_USER}
-su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${HADOOP_MAPREDUCE_USER}:system /mapred" ${HADOOP_HDFS_USER}
-
-if [ $? -eq 0 ]; then
-  echo "Completed."
-else
-  echo "Unknown error occurred, check hadoop logs for details."
-fi
-
-echo
-echo "Please startup datanode processes: /etc/init.d/hadoop-datanode start"

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-single-node.sh

@@ -1,219 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script for setup HDFS file system for single node deployment
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     --default                   Setup system as default
-     -h                          Display this message
-  "
-  exit 1
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'default' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-if [ -e /etc/hadoop/hadoop-env.sh ]; then
-  . /etc/hadoop/hadoop-env.sh
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --default)
-      AUTOMATED=1; shift
-      ;;
-    -h)
-      usage
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-if [ "${AUTOMATED}" != "1" ]; then
-  echo "Welcome to Hadoop single node setup wizard"
-  echo
-  echo -n "Would you like to use default single node configuration? (y/n) "
-  read SET_CONFIG
-  echo -n "Would you like to format name node? (y/n) "
-  read SET_FORMAT
-  echo -n "Would you like to setup default directory structure? (y/n) "
-  read SET_MKDIR
-  echo -n "Would you like to start up Hadoop? (y/n) "
-  read STARTUP
-  echo -n "Would you like to start up Hadoop on reboot? (y/n) "
-  read SET_REBOOT
-  echo
-  echo "Review your choices:"
-  echo
-  echo "Setup single node configuration    : ${SET_CONFIG}"
-  echo "Format namenode                    : ${SET_FORMAT}"
-  echo "Setup default file system structure: ${SET_MKDIR}"
-  echo "Start up Hadoop                    : ${STARTUP}"
-  echo "Start up Hadoop on reboot          : ${SET_REBOOT}"
-  echo
-  echo -n "Proceed with setup? (y/n) "
-  read CONFIRM
-  if [ "${CONFIRM}" != "y" ]; then
-    echo "User aborted setup, exiting..."
-    exit 1
-  fi
-else
-  SET_CONFIG="y"
-  SET_FORMAT="y"
-  SET_MKDIR="y"
-  STARTUP="y"
-  SET_REBOOT="y"
-fi
-
-AUTOMATED=${AUTOMATED:-0}
-SET_CONFIG=${SET_CONFIG:-y}
-SET_FORMAT=${SET_FORMAT:-n}
-SET_MKDIR=${SET_MKDIR:-y}
-STARTUP=${STARTUP:-y}
-SET_REBOOT=${SET_REBOOT:-y}
-
-# Make sure system is not already started
-/etc/init.d/hadoop-namenode stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-datanode stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null
-/etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null
-
-if [ "${SET_CONFIG}" == "y" ]; then
-  JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-  HADOOP_NN_HOST=${HADOOP_NN_HOST:-localhost}
-  HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
-  HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-  HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost}
-  HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs}
-  HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
-  HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
-  HADOOP_LOG_DIR="/var/log/hadoop"
-  HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
-  HADOOP_REPLICATION=${HADOOP_RELICATION:-1}
-  ${HADOOP_PREFIX}/sbin/hadoop-setup-conf.sh --auto \
-    --hdfs-user=hdfs \
-    --mapreduce-user=mapred \
-    --conf-dir=${HADOOP_CONF_DIR} \
-    --datanode-dir=${HADOOP_DN_DIR} \
-    --hdfs-dir=${HADOOP_HDFS_DIR} \
-    --jobtracker-host=${HADOOP_JT_HOST} \
-    --log-dir=${HADOOP_LOG_DIR} \
-    --pid-dir=${HADOOP_PID_DIR} \
-    --mapred-dir=${HADOOP_MAPRED_DIR} \
-    --namenode-dir=${HADOOP_NN_DIR} \
-    --namenode-host=${HADOOP_NN_HOST} \
-    --replication=${HADOOP_REPLICATION}
-fi
-
-if [ ! -e ${HADOOP_NN_DIR} ]; then
-  rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
-  mkdir -p ${HADOOP_HDFS_DIR}
-  chmod 755 ${HADOOP_HDFS_DIR}
-  chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  /etc/init.d/hadoop-namenode format
-elif [ "${SET_FORMAT}" == "y" ]; then
-  rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
-  mkdir -p ${HADOOP_HDFS_DIR}
-  chmod 755 ${HADOOP_HDFS_DIR}
-  chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  rm -rf ${HADOOP_NN_DIR}
-  /etc/init.d/hadoop-namenode format
-fi
-
-/etc/init.d/hadoop-namenode start
-/etc/init.d/hadoop-datanode start
-
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown mapred:mapred /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp' hdfs
-
-/etc/init.d/hadoop-jobtracker start
-/etc/init.d/hadoop-tasktracker start
-
-if [ "${SET_REBOOT}" == "y" ]; then
-  if [ -e /etc/debian_version ]; then
-    ln -sf ../init.d/hadoop-namenode /etc/rc2.d/S90hadoop-namenode
-    ln -sf ../init.d/hadoop-datanode /etc/rc2.d/S91hadoop-datanode
-    ln -sf ../init.d/hadoop-jobtracker /etc/rc2.d/S92hadoop-jobtracker
-    ln -sf ../init.d/hadoop-tasktracker /etc/rc2.d/S93hadoop-tasktracker
-    ln -sf ../init.d/hadoop-namenode /etc/rc6.d/S10hadoop-namenode
-    ln -sf ../init.d/hadoop-datanode /etc/rc6.d/S11hadoop-datanode
-    ln -sf ../init.d/hadoop-jobtracker /etc/rc6.d/S12hadoop-jobtracker
-    ln -sf ../init.d/hadoop-tasktracker /etc/rc6.d/S13hadoop-tasktracker
-  elif [ -e /etc/redhat-release ]; then
-    /sbin/chkconfig hadoop-namenode --add
-    /sbin/chkconfig hadoop-datanode --add
-    /sbin/chkconfig hadoop-jobtracker --add
-    /sbin/chkconfig hadoop-tasktracker --add
-    /sbin/chkconfig hadoop-namenode on
-    /sbin/chkconfig hadoop-datanode on
-    /sbin/chkconfig hadoop-jobtracker on
-    /sbin/chkconfig hadoop-tasktracker on
-  fi
-fi
-
-if [ "${STARTUP}" != "y" ]; then
-  /etc/init.d/hadoop-namenode stop
-  /etc/init.d/hadoop-datanode stop
-  /etc/init.d/hadoop-jobtracker stop
-  /etc/init.d/hadoop-tasktracker stop
-fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-validate-setup.sh

@@ -1,183 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-
-###############################################################################
-# Run the following jobs to validate a hadoop cluster
-## teragen
-## terasort
-## teravalidate
-# If they all pass 0 will be returned and 1 otherwise
-# The test will work for both secure and unsecure deploys. If the kerberos-realm
-# is passed we will assume that the deploy is secure and proceed with a kinit before
-# running the validation jobs.
-################################################################################
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-DEFAULT_LIBEXEC_DIR="$bin"/../libexec
-HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-
-usage() {
-  echo "
-usage: $0 <parameters>
-
-  Optional parameters:
-     -h                                                              Display this message
-     --user=hdfs
-     --user_keytab=/home/hdfs/hdfs.keytab
-     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
-  "
-  exit 1
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'user:' \
-  -l 'user-keytab:' \
-  -l 'kerberos-realm:' \
-  -o 'h' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --user)
-      TEST_USER=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --user-keytab)
-      USER_KEYTAB_FILE=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --kerberos-realm)
-      KERBEROS_REALM=$2; shift 2
-      AUTOMATED=1
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-#set the hadoop command and the path to the hadoop examples jar
-HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
-
-#find the hadoop examples jar
-HADOOP_EXAMPLES_JAR=''
-
-#find under HADOOP_PREFIX (tar ball install)
-HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-mapreduce-examples-*.jar' | head -n1`
-
-#if its not found look under /usr/share/hadoop (rpm/deb installs)
-if [ "$HADOOP_EXAMPLES_JAR" == '' ]
-then
-  HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-mapreduce-examples-*.jar' | head -n1`
-fi
-
-#if it is still empty then dont run the tests
-if [ "$HADOOP_EXAMPLES_JAR" == '' ]
-then
-  echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
-  exit 1
-fi
-
-# do a kinit if secure
-if [ "${KERBEROS_REALM}" != "" ]; then
-  # Determine kerberos location base on Linux distro.
-  if [ -e /etc/lsb-release ]; then
-    KERBEROS_BIN=/usr/bin
-  else
-    KERBEROS_BIN=/usr/kerberos/bin
-  fi
-  kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
-  echo $kinit_cmd
-  eval $kinit_cmd
-  if [ $? -ne 0 ]
-  then
-    echo "kinit command did not run successfully."
-    exit 1
-  fi
-fi
-
-
-#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
-PARENT_DIR="validate_deploy_`date +%s`"
-TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
-TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
-TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
-#tera gen cmd
-TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
-
-#tera sort cmd
-TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
-
-#tera validate cmd
-TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
-
-echo "Starting teragen...."
-
-#run tera gen
-echo $TERA_GEN_CMD
-eval $TERA_GEN_CMD
-if [ $? -ne 0 ]; then
-  echo "tera gen failed."
-  exit 1
-fi
-
-echo "Teragen passed starting terasort...."
-
-
-#run tera sort
-echo $TERA_SORT_CMD
-eval $TERA_SORT_CMD
-if [ $? -ne 0 ]; then
-  echo "tera sort failed."
-  exit 1
-fi
-
-echo "Terasort passed starting teravalidate...."
-
-#run tera validate
-echo $TERA_VALIDATE_CMD
-eval $TERA_VALIDATE_CMD
-if [ $? -ne 0 ]; then
-  echo "tera validate failed."
-  exit 1
-fi
-
-echo "teragen, terasort, teravalidate passed."
-echo "Cleaning the data created by tests: $PARENT_DIR"
-
-CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
-echo $CLEANUP_CMD
-eval $CLEANUP_CMD
-
-exit 0

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-datanode

@@ -1,93 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop datanode
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop datanode
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid"
-desc="Hadoop datanode daemon"
-HADOOP_PREFIX="/usr"
-
-start() {
-  echo -n $"Starting $desc (hadoop-datanode): "
-  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
-  else
-    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
-  fi
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-datanode
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-datanode): "
-  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
-    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
-  else
-    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
-  fi
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-datanode $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-datanode ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-jobtracker

@@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop jobtracker
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop jobtracker
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid"
-desc="Hadoop jobtracker daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-jobtracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start jobtracker
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-jobtracker
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-jobtracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop jobtracker
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-jobtracker $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-jobtracker ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-namenode

@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop namenode
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop namenode
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid"
-desc="Hadoop namenode daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-namenode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start namenode $1
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-namenode
-  return $RETVAL
-}
-
-upgrade() {
-  start -upgrade
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-namenode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop namenode
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-namenode $PIDFILE
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-restart() {
-  stop
-  start
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-namenode ] && restart || :
-}
-
-format() {
-  daemon --user hdfs ${HADOOP_PREFIX}/bin/hadoop namenode -format
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  upgrade)
-    upgrade
-    ;;
-  format)
-    format
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart|try-restart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|try-restart|upgrade}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-tasktracker

@@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# 
-# Starts a Hadoop tasktracker
-# 
-# chkconfig: 2345 90 10
-# description: Hadoop tasktracker
-
-source /etc/rc.d/init.d/functions
-source /etc/default/hadoop-env.sh
-
-RETVAL=0
-PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid"
-desc="Hadoop tasktracker daemon"
-export HADOOP_PREFIX=${HADOOP_PREFIX:-/usr}
-
-start() {
-  echo -n $"Starting $desc (hadoop-tasktracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start tasktracker
-  RETVAL=$?
-  echo
-  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-tasktracker
-  return $RETVAL
-}
-
-stop() {
-  echo -n $"Stopping $desc (hadoop-tasktracker): "
-  daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop tasktracker
-  RETVAL=$?
-  sleep 5
-  echo
-  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-tasktracker $PIDFILE
-}
-
-restart() {
-  stop
-  start
-}
-
-checkstatus(){
-  status -p $PIDFILE ${JAVA_HOME}/bin/java
-  RETVAL=$?
-}
-
-condrestart(){
-  [ -e /var/lock/subsys/hadoop-tasktracker ] && restart || :
-}
-
-case "$1" in
-  start)
-    start
-    ;;
-  stop)
-    stop
-    ;;
-  status)
-    checkstatus
-    ;;
-  restart)
-    restart
-    ;;
-  condrestart)
-    condrestart
-    ;;
-  *)
-    echo $"Usage: $0 {start|stop|status|restart|condrestart}"
-    exit 1
-esac
-
-exit $RETVAL

hadoop-common-project/hadoop-common/src/main/packages/rpm/spec/hadoop.spec

@@ -1,174 +0,0 @@
-#   Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  See the NOTICE file distributed with
-#   this work for additional information regarding copyright ownership.
-#   The ASF licenses this file to You under the Apache License, Version 2.0
-#   (the "License"); you may not use this file except in compliance with
-#   the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#
-# RPM Spec file for Hadoop version @version@
-#
-
-%define name         hadoop-common
-%define version      @version@
-%define release      @package.release@
-
-# Installation Locations
-%define _prefix      @package.prefix@
-%define _bin_dir     %{_prefix}/bin
-%define _conf_dir    @package.conf.dir@
-%define _lib_dir     %{_prefix}/lib
-%define _lib64_dir   %{_prefix}/lib64
-%define _libexec_dir %{_prefix}/libexec
-%define _log_dir     @package.log.dir@
-%define _pid_dir     @package.pid.dir@
-%define _sbin_dir    %{_prefix}/sbin
-%define _share_dir   %{_prefix}/share
-%define _var_dir     @package.var.dir@
-
-# Build time settings
-%define _build_dir  @package.build.dir@
-%define _final_name @final.name@
-%define debug_package %{nil}
-
-# Disable brp-java-repack-jars for aspect J
-%define __os_install_post    \
-    /usr/lib/rpm/redhat/brp-compress \
-    %{!?__debug_package:/usr/lib/rpm/redhat/brp-strip %{__strip}} \
-    /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
-    /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \
-    /usr/lib/rpm/brp-python-bytecompile %{nil}
-
-# RPM searches perl files for dependancies and this breaks for non packaged perl lib
-# like thrift so disable this
-%define _use_internal_dependency_generator 0
-
-%ifarch i386
-%global hadoop_arch Linux-i386-32
-%endif
-%ifarch amd64 x86_64
-%global hadoop_arch Linux-amd64-64
-%endif
-%ifarch noarch
-%global hadoop_arch ""
-%endif
-
-Summary: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing
-License: Apache License, Version 2.0
-URL: http://hadoop.apache.org/core/
-Vendor: Apache Software Foundation
-Group: Development/Libraries
-Name: %{name}
-Version: %{version}
-Release: %{release} 
-Source0: %{_final_name}-bin.tar.gz
-Prefix: %{_prefix}
-Prefix: %{_conf_dir}
-Prefix: %{_log_dir}
-Prefix: %{_pid_dir}
-Buildroot: %{_build_dir}
-Requires: sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, jdk >= 1.6
-AutoReqProv: no
-Provides: hadoop
-
-%description
-The Apache Hadoop project develops open-source software for reliable, scalable, 
-distributed computing.  Hadoop includes these subprojects:
-
-Hadoop Common: The common utilities that support the other Hadoop subprojects.
-
-%prep
-%setup -n %{_final_name}
-
-%build
-if [ -d ${RPM_BUILD_DIR}%{_prefix} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_prefix}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_log_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_log_dir}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_conf_dir}
-fi
-
-if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then
-  rm -rf ${RPM_BUILD_DIR}%{_pid_dir}
-fi
-
-mkdir -p ${RPM_BUILD_DIR}%{_prefix}
-mkdir -p ${RPM_BUILD_DIR}%{_bin_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_lib_dir}
-%ifarch amd64 x86_64
-mkdir -p ${RPM_BUILD_DIR}%{_lib64_dir}
-%endif
-mkdir -p ${RPM_BUILD_DIR}%{_libexec_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_log_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_conf_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_pid_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_sbin_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_share_dir}
-mkdir -p ${RPM_BUILD_DIR}%{_var_dir}
-
-#########################
-#### INSTALL SECTION ####
-#########################
-%install
-mv ${RPM_BUILD_DIR}/%{_final_name}/bin/* ${RPM_BUILD_DIR}%{_bin_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/etc/hadoop/* ${RPM_BUILD_DIR}%{_conf_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/lib/* ${RPM_BUILD_DIR}%{_lib_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/libexec/* ${RPM_BUILD_DIR}%{_libexec_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/sbin/* ${RPM_BUILD_DIR}%{_sbin_dir}
-mv ${RPM_BUILD_DIR}/%{_final_name}/share/* ${RPM_BUILD_DIR}%{_share_dir}
-rm -rf ${RPM_BUILD_DIR}/%{_final_name}/etc
-
-%pre
-getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop
-
-%post
-bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \
-       --prefix=${RPM_INSTALL_PREFIX0} \
-       --bin-dir=${RPM_INSTALL_PREFIX0}/bin \
-       --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \
-       --conf-dir=${RPM_INSTALL_PREFIX1} \
-       --log-dir=${RPM_INSTALL_PREFIX2} \
-       --pid-dir=${RPM_INSTALL_PREFIX3}
-
-%preun
-bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \
-       --prefix=${RPM_INSTALL_PREFIX0} \
-       --bin-dir=${RPM_INSTALL_PREFIX0}/bin \
-       --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \
-       --conf-dir=${RPM_INSTALL_PREFIX1} \
-       --log-dir=${RPM_INSTALL_PREFIX2} \
-       --pid-dir=${RPM_INSTALL_PREFIX3} \
-       --uninstall
-
-%files 
-%defattr(-,root,root)
-%attr(0755,root,hadoop) %{_log_dir}
-%attr(0775,root,hadoop) %{_pid_dir}
-%config(noreplace) %{_conf_dir}/configuration.xsl
-%config(noreplace) %{_conf_dir}/core-site.xml
-%config(noreplace) %{_conf_dir}/hadoop-env.sh
-%config(noreplace) %{_conf_dir}/hadoop-metrics.properties
-%config(noreplace) %{_conf_dir}/hadoop-metrics2.properties
-%config(noreplace) %{_conf_dir}/hadoop-policy.xml
-%config(noreplace) %{_conf_dir}/log4j.properties
-%config(noreplace) %{_conf_dir}/masters
-%config(noreplace) %{_conf_dir}/slaves
-%config{noreplace) %{_conf_dir}/fair-scheduler.xml
-%{_conf_dir}/hadoop-env.sh.template
-%{_conf_dir}/ssl-client.xml.example
-%{_conf_dir}/ssl-server.xml.example
-%{_prefix}
-

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/capacity-scheduler.xml

@@ -1,198 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<!-- This is the configuration file for the resource manager in Hadoop. -->
-<!-- You can configure various scheduling parameters related to queues. -->
-<!-- The properties for a queue follow a naming convention,such as, -->
-<!-- mapred.capacity-scheduler.queue.<queue-name>.property-name. -->
-
-<configuration>
-
-  <property>
-    <name>mapred.capacity-scheduler.maximum-system-jobs</name>
-    <value>3000</value>
-    <description>Maximum number of jobs in the system which can be initialized,
-     concurrently, by the CapacityScheduler.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.capacity</name>
-    <value>100</value>
-    <description>Percentage of the number of slots in the cluster that are
-      to be available for jobs in this queue.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-capacity</name>
-    <value>-1</value>
-    <description>
-	maximum-capacity defines a limit beyond which a queue cannot use the capacity of the cluster.
-	This provides a means to limit how much excess capacity a queue can use. By default, there is no limit.
-	The maximum-capacity of a queue can only be greater than or equal to its minimum capacity.
-        Default value of -1 implies a queue can use complete capacity of the cluster.
-
-        This property could be to curtail certain jobs which are long running in nature from occupying more than a 
-        certain percentage of the cluster, which in the absence of pre-emption, could lead to capacity guarantees of 
-        other queues being affected.
-        
-        One important thing to note is that maximum-capacity is a percentage , so based on the cluster's capacity
-        the max capacity would change. So if large no of nodes or racks get added to the cluster , max Capacity in 
-        absolute terms would increase accordingly.
-    </description>    
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.minimum-user-limit-percent</name>
-    <value>100</value>
-    <description> Each queue enforces a limit on the percentage of resources 
-    allocated to a user at any given time, if there is competition for them. 
-    This user limit can vary between a minimum and maximum value. The former
-    depends on the number of users who have submitted jobs, and the latter is
-    set to this property value. For example, suppose the value of this 
-    property is 25. If two users have submitted jobs to a queue, no single 
-    user can use more than 50% of the queue resources. If a third user submits
-    a job, no single user can use more than 33% of the queue resources. With 4 
-    or more users, no user can use more than 25% of the queue's resources. A 
-    value of 100 implies no user limits are imposed. 
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.user-limit-factor</name>
-    <value>1</value>
-    <description>The multiple of the queue capacity which can be configured to 
-    allow a single user to acquire more slots. 
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks</name>
-    <value>200000</value>
-    <description>The maximum number of tasks, across all jobs in the queue, 
-    which can be initialized concurrently. Once the queue's jobs exceed this 
-    limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user</name>
-    <value>100000</value>
-    <description>The maximum number of tasks per-user, across all the of the 
-    user's jobs in the queue, which can be initialized concurrently. Once the 
-    user's jobs exceed this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.queue.default.init-accept-jobs-factor</name>
-    <value>10</value>
-    <description>The multipe of (maximum-system-jobs * queue-capacity) used to 
-    determine the number of jobs which are accepted by the scheduler.  
-    </description>
-  </property>
-
-  <!-- The default configuration settings for the capacity task scheduler -->
-  <!-- The default values would be applied to all the queues which don't have -->
-  <!-- the appropriate property for the particular queue -->
-  <property>
-    <name>mapred.capacity-scheduler.default-supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions by default in a job queue.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
-    <value>100</value>
-    <description>The percentage of the resources limited to a particular user
-      for the job queue at any given point of time by default.
-    </description>
-  </property>
-
-
-  <property>
-    <name>mapred.capacity-scheduler.default-user-limit-factor</name>
-    <value>1</value>
-    <description>The default multiple of queue-capacity which is used to 
-    determine the amount of slots a single user can consume concurrently.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-queue</name>
-    <value>200000</value>
-    <description>The default maximum number of tasks, across all jobs in the 
-    queue, which can be initialized concurrently. Once the queue's jobs exceed 
-    this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-user</name>
-    <value>100000</value>
-    <description>The default maximum number of tasks per-user, across all the of 
-    the user's jobs in the queue, which can be initialized concurrently. Once 
-    the user's jobs exceed this limit they will be queued on disk.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-init-accept-jobs-factor</name>
-    <value>10</value>
-    <description>The default multipe of (maximum-system-jobs * queue-capacity) 
-    used to determine the number of jobs which are accepted by the scheduler.  
-    </description>
-  </property>
-
-  <!-- Capacity scheduler Job Initialization configuration parameters -->
-  <property>
-    <name>mapred.capacity-scheduler.init-poll-interval</name>
-    <value>5000</value>
-    <description>The amount of time in miliseconds which is used to poll 
-    the job queues for jobs to initialize.
-    </description>
-  </property>
-  <property>
-    <name>mapred.capacity-scheduler.init-worker-threads</name>
-    <value>5</value>
-    <description>Number of worker threads which would be used by
-    Initialization poller to initialize jobs in a set of queue.
-    If number mentioned in property is equal to number of job queues
-    then a single thread would initialize jobs in a queue. If lesser
-    then a thread would get a set of queues assigned. If the number
-    is greater then number of threads would be equal to number of 
-    job queues.
-    </description>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/commons-logging.properties

@@ -1,25 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#Logging Implementation
-
-#Log4J
-org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
-
-#JDK Logger
-#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/core-site.xml

@@ -1,100 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
-  <property>
-    <name>local.realm</name>
-    <value>${KERBEROS_REALM}</value>
-  </property>
-
-  <!-- file system properties -->
-
-  <property>
-    <name>fs.default.name</name>
-    <value>hdfs://${HADOOP_NN_HOST}:8020</value>
-    <description>The name of the default file system.  Either the
-      literal string "local" or a host:port for NDFS.
-    </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>fs.trash.interval</name>
-    <value>360</value>
-    <description>Number of minutes between trash checkpoints.
-      If zero, the trash feature is disabled.
-    </description>
-  </property>
-  
-  <property>
-    <name>hadoop.security.auth_to_local</name>
-    <value>
-	RULE:[2:$1@$0]([jt]t@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0]([nd]n@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](mapred@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0](hdfs@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](mapredqa@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
-        RULE:[2:$1@$0](hdfsqa@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
-        RULE:[2:$1@$0](hm@.*${KERBEROS_REALM})s/.*/${HBASE_USER}/
-        RULE:[2:$1@$0](rs@.*${KERBEROS_REALM})s/.*/${HBASE_USER}/
-        DEFAULT
-    </value>
-    <description></description>
-  </property>
-
-  <property>
-    <name>hadoop.security.authentication</name>
-    <value>${SECURITY_TYPE}</value>
-    <description>
-      Set the authentication for the cluster. Valid values are: simple or
-      kerberos.
-    </description>
-  </property>
-
-  <property>
-    <name>hadoop.security.authorization</name>
-    <value>${SECURITY}</value>
-    <description>
-      Enable authorization for different protocols.
-    </description>
-  </property>
-
-  <property>
-    <name>hadoop.security.groups.cache.secs</name>
-    <value>14400</value>
-  </property>
-
-  <property>
-    <name>hadoop.kerberos.kinit.command</name>
-    <value>${KINIT}</value>
-  </property>
-
-  <property>
-    <name>hadoop.http.filter.initializers</name>
-    <value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh

@@ -1,78 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Set Hadoop-specific environment variables here.
-
-# The only required environment variable is JAVA_HOME.  All others are
-# optional.  When running a distributed configuration it is best to
-# set JAVA_HOME in this file, so that it is correctly defined on
-# remote nodes.
-
-# The java implementation to use.
-export JAVA_HOME=${JAVA_HOME}
-
-# The jsvc implementation to use. Jsvc is required to run secure datanodes.
-#export JSVC_HOME=${JSVC_HOME}
-
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
-
-# Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
-for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
-  if [ "$HADOOP_CLASSPATH" ]; then
-    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
-  else
-    export HADOOP_CLASSPATH=$f
-  fi
-done
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-#export HADOOP_HEAPSIZE=
-#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
-
-# Extra Java runtime options.  Empty by default.
-export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS"
-
-# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
-export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
-
-export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
-
-# The ZKFC does not need a large heap, and keeping it small avoids
-# any potential for long GC pauses
-export HADOOP_ZKFC_OPTS="-Xmx256m $HADOOP_ZKFC_OPTS"
-
-# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS"
-#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
-
-# On secure datanodes, user to run the datanode as after dropping privileges
-export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
-
-# Where log files are stored.  $HADOOP_HOME/logs by default.
-#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
-
-# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
-
-# The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=${HADOOP_PID_DIR}
-export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
-
-# A string representing this instance of hadoop. $USER by default.
-export HADOOP_IDENT_STRING=$USER

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-metrics2.properties

@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# syntax: [prefix].[source|sink|jmx].[instance].[options]
-# See package.html for org.apache.hadoop.metrics2 for details
-
-*.period=60
-

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml

@@ -1,250 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-  <property>
-    <name>security.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientProtocol, which is used by user code
-    via the DistributedFileSystem.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.client.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
-    for block recovery.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for DatanodeProtocol, which is used by datanodes to
-    communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.inter.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
-    for updating generation timestamp.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.namenode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for NamenodeProtocol, the protocol used by the secondary
-    namenode to communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.inter.tracker.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterTrackerProtocol, used by the tasktrackers to
-    communicate with the jobtracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
- 
-  <property>
-    <name>security.job.submission.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for JobSubmissionProtocol, used by job clients to
-    communciate with the jobtracker for job submission, querying job status etc.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.task.umbilical.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
-    tasks to communicate with the parent tasktracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
- <property>
-    <name>security.admin.operations.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for AdminOperationsProtocol. Used for admin commands.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.usertogroups.mappings.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for RefreshUserMappingsProtocol. Used to refresh
-    users mappings. The ACL is a comma-separated list of user and
-    group names. The user and group list is separated by a blank. For
-    e.g. "alice,bob users,wheel".  A special value of "*" means all
-    users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.policy.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
-    dfsadmin and mradmin commands to refresh the security policy in-effect.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-
-  <!-- YARN Protocols -->
-
-  <property>
-    <name>security.resourcetracker.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
-    <description>ACL for ResourceTracker protocol, used by the
-    ResourceManager and NodeManager to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.admin.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
-    <description>ACL for RMAdminProtocol, for admin commands. 
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.client.resourcemanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientRMProtocol, used by the ResourceManager 
-    and applications submission clients to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationmaster.resourcemanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for AMRMProtocol, used by the ResourceManager 
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.containermanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ContainerManager protocol, used by the NodeManager 
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.resourcelocalizer.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ResourceLocalizer protocol, used by the NodeManager 
-    and ResourceLocalizer to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.task.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
-    tasks to communicate with the parent tasktracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for MRClientProtocol, used by job clients to
-    communciate with the MR ApplicationMaster to query job status etc. 
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-  
-  <property>
-    <name>security.ha.service.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for HAService protocol used by HAAdmin to manage the
-      active and stand-by states of namenode.</description>
-  </property>
-  <property>
-    <name>security.zkfc.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for access to the ZK Failover Controller
-    </description>
-  </property>
-
-   <property>
-      <name>security.mrhs.client.protocol.acl</name>
-      <value>*</value>
-      <description>ACL for HSClientProtocol, used by job clients to
-       communciate with the MR History Server job status etc. 
-       The ACL is a comma-separated list of user and group names. The user and
-       group list is separated by a blank. For e.g. "alice,bob users,wheel".
-       A special value of "*" means all users are allowed.</description>
-    </property>
-  
-  <property>
-    <name>security.qjournal.service.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for QJournalProtocol, used by the NN to communicate with
-    JNs when using the QuorumJournalManager for edit logs.</description>
-  </property>
-
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml

@@ -1,269 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<configuration>
-
-<!-- file system properties -->
-
-  <property>
-    <name>dfs.namenode.name.dir</name>
-    <value>${HADOOP_NN_DIR}</value>
-    <description>Determines where on the local filesystem the DFS name node
-      should store the name table.  If this is a comma-delimited list
-      of directories then the name table is replicated in all of the
-      directories, for redundancy. </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>dfs.datanode.data.dir</name>
-    <value>${HADOOP_DN_DIR}</value>
-    <description>Determines where on the local filesystem an DFS data node
-       should store its blocks.  If this is a comma-delimited
-       list of directories, then data will be stored in all named
-       directories, typically on different devices.
-       Directories that do not exist are ignored.
-    </description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>dfs.namenode.safemode.threshold-pct</name>
-    <value>1.0f</value>
-    <description>
-        Specifies the percentage of blocks that should satisfy the minimal 
-        replication requirement defined by dfs.namenode.replication.min. 
-        Values less than or equal to 0 mean not to start in safe mode.
-        Values greater than 1 will make safe mode permanent.
-        </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.address</name>
-    <value>${HADOOP_DN_ADDR}</value>
-  </property>
-
-  <property>
-    <name>dfs.datanode.http.address</name>
-    <value>${HADOOP_DN_HTTP_ADDR}</value>
-  </property>
-
-  <property>
-    <name>dfs.namenode.http-address</name>
-    <value>${HADOOP_NN_HOST}:50070</value>
-    <description>The name of the default file system.  Either the
-       literal string "local" or a host:port for NDFS.
-    </description>
-    <final>true</final>
-  </property>
-
-  <!-- Permissions configuration -->
-  <property>
-    <name>dfs.umaskmode</name>
-    <value>077</value>
-    <description>
-      The octal umask used when creating files and directories.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.block.access.token.enable</name>
-    <value>${SECURITY}</value>
-    <description>
-      Are access tokens are used as capabilities for accessing datanodes.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.kerberos.principal</name>
-    <value>nn/_HOST@${local.realm}</value>
-    <description>
-      Kerberos principal name for the NameNode
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.kerberos.principal</name>
-    <value>nn/_HOST@${local.realm}</value>
-    <description>
-        Kerberos principal name for the secondary NameNode.
-    </description>
-  </property>
-
-
-  <property>
-    <name>dfs.namenode.kerberos.https.principal</name>
-    <value>host/_HOST@${local.realm}</value>
-    <description>
-       The Kerberos principal for the host that the NameNode runs on.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.kerberos.https.principal</name>
-    <value>host/_HOST@${local.realm}</value>
-    <description>
-      The Kerberos principal for the hostthat the secondary NameNode runs on.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.kerberos.principal</name>
-    <value>dn/_HOST@${local.realm}</value>
-    <description>
-      The Kerberos principal that the DataNode runs as. "_HOST" is replaced by 
-      the real host name.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.web.authentication.kerberos.principal</name>
-    <value>HTTP/_HOST@${local.realm}</value>
-    <description>
-      The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
-
-      The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos
-      HTTP SPNEGO specification.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.web.authentication.kerberos.keytab</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      The Kerberos keytab file with the credentials for the
-      HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.keytab.file</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      Combined keytab file containing the namenode service and host principals.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.secondary.namenode.keytab.file</name>
-    <value>/etc/security/keytabs/nn.service.keytab</value>
-    <description>
-      Combined keytab file containing the namenode service and host principals.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.keytab.file</name>
-    <value>/etc/security/keytabs/dn.service.keytab</value>
-    <description>
-        The filename of the keytab file for the DataNode.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.https.port</name>
-    <value>50470</value>
-    <description>The https port where namenode binds</description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.https-address</name>
-    <value>${HADOOP_NN_HOST}:50470</value>
-    <description>The https address where namenode binds</description>
-  </property>
-
-  <property>
-    <name>dfs.datanode.data.dir.perm</name>
-    <value>${DFS_DATANODE_DIR_PERM}</value>
-    <description>The permissions that should be there on dfs.datanode.data.dir
-      directories. The datanode will not come up if the permissions are
-      different on existing dfs.datanode.data.dir directories. If the directories
-      don't exist, they will be created with this permission.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.cluster.administrators</name>
-    <value>${HADOOP_HDFS_USER}</value>
-    <description>ACL for who all can view the default servlets in the HDFS</description>
-  </property>
-  
-  <property>
-    <name>dfs.permissions.superusergroup</name>
-    <value>${HADOOP_GROUP}</value>
-    <description>The name of the group of super-users.</description>
-  </property>
-  
-  <property>
-    <name>dfs.namenode.http-address</name>
-    <value>${HADOOP_NN_HOST}:50070</value>
-    <description>
-      The address and the base port where the dfs namenode web ui will listen on.
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.namenode.https-address</name>
-    <value>${HADOOP_NN_HOST}:50470</value>
-  </property>
-
-  <property>
-    <name>dfs.namenode.secondary.http-address</name>
-    <value>${HADOOP_SNN_HOST}:50090</value>
-    <description>
-      The secondary namenode http server address and port.
-      If the port is 0 then the server will start on a free port.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.hosts</name>
-    <value>${HADOOP_CONF_DIR}/dfs.include</value>
-    <description>Names a file that contains a list of hosts that are
-      permitted to connect to the namenode. The full pathname of the file
-      must be specified.  If the value is empty, all hosts are
-      permitted.</description>
-  </property>
-
-  <property>
-    <name>dfs.hosts.exclude</name>
-    <value>${HADOOP_CONF_DIR}/dfs.exclude</value>
-    <description>Names a file that contains a list of hosts that are
-      not permitted to connect to the namenode.  The full pathname of the
-      file must be specified.  If the value is empty, no hosts are
-      excluded.
-    </description>
-  </property>
-  <property>
-    <name>dfs.webhdfs.enabled</name>
-    <value>${DFS_WEBHDFS_ENABLED}</value>
-    <description>Enable or disable webhdfs. Defaults to false</description>
-  </property>
-  <property>
-    <name>dfs.support.append</name>
-    <value>${DFS_SUPPORT_APPEND}</value>
-    <description>Enable or disable append. Defaults to false</description>
-  </property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/log4j.properties

@@ -1,212 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=ALL
-
-# Null Appender
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Rolling File Appender - cap space usage at 5gb.
-#
-hadoop.log.maxfilesize=256MB
-hadoop.log.maxbackupindex=20
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
-log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this 
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-#Security appender
-#
-hadoop.security.logger=INFO,NullAppender
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth-${user.name}.audit
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# Daily Rolling Security appender
-#
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,NullAppender
-hdfs.audit.log.maxfilesize=256MB
-hdfs.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
-log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,NullAppender
-mapred.audit.log.maxfilesize=256MB
-mapred.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
-log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-#
-# Job Summary Appender 
-#
-# Use following logger to send summary to separate file defined by 
-# hadoop.mapreduce.jobsummary.log.file :
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-# 
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
-hadoop.mapreduce.jobsummary.log.maxbackupindex=20
-log4j.appender.JSA=org.apache.log4j.RollingFileAppender
-log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
-log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
-log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
-#
-# Yarn ResourceManager Application Summary Log 
-#
-# Set the ResourceManager summary log filename
-#yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
-# Set the ResourceManager summary log level and appender
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-#    - hadoop.log.dir (Hadoop Log directory)
-#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-
-#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
-#log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-#log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
-#log4j.appender.RMSUMMARY.MaxFileSize=256MB
-#log4j.appender.RMSUMMARY.MaxBackupIndex=20
-#log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-#log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-queue-acls.xml

@@ -1,33 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-<configuration>
-<property>
-<name>mapred.queue.default.acl-submit-job</name>
-<value>*</value>
-</property>
-<property>
-<name>mapred.queue.default.acl-administer-jobs</name>
-<value>*</value>
-</property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml

@@ -1,308 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Copyright 2011 The Apache Software Foundation
- 
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- 
-  <property>
-    <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
-    <value>250</value>
-    <description>Normally, this is the amount of time before killing
-      processes, and the recommended-default is 5.000 seconds - a value of
-      5000 here.  In this case, we are using it solely to blast tasks before
-      killing them, and killing them very quickly (1/4 second) to guarantee
-      that we do not leave VMs around for later jobs.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.system.dir</name>
-    <value>/mapred/mapredsystem</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker</name>
-    <value>${HADOOP_JT_HOST}:9000</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.http.address</name>
-    <value>${HADOOP_JT_HOST}:50030</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.local.dir</name>
-    <value>${HADOOP_MAPRED_DIR}</value>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.administrators</name>
-    <value>${HADOOP_MR_USER}</value>
-  </property>
-
-  <property>
-    <name>mapred.map.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some map tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.reduce.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some reduce tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.output.compression.type</name>
-    <value>BLOCK</value>
-    <description>If the job outputs are to compressed as SequenceFiles, how 
-       should they be compressed? Should be one of NONE, RECORD or BLOCK.
-    </description>
-  </property>
-
-  <property>
-    <name>jetty.connector</name>
-    <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
-  </property>
-
-  <property>
-    <name>mapred.task.tracker.task-controller</name>
-    <value>${TASK_CONTROLLER}</value>
-  </property>
-
-  <property>
-    <name>mapred.child.root.logger</name>
-    <value>INFO,TLA</value>
-  </property>
-
-  <property>
-    <name>mapred.child.java.opts</name>
-    <value>-server -Xmx640m -Djava.net.preferIPv4Stack=true</value>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.persist.jobstatus.active</name>
-    <value>true</value>
-    <description>Indicates if persistency of job status information is
-      active or not.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.persist.jobstatus.dir</name>
-    <value>file:///${HADOOP_LOG_DIR}/${HADOOP_MR_USER}/jobstatus</value>
-    <description>The directory where the job status information is persisted
-      in a file system to be available after it drops of the memory queue and
-      between jobtracker restarts.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.history.completed.location</name>
-    <value>/mapred/history/done</value>
-  </property>
-
-  <property>
-    <name>mapred.heartbeats.in.second</name>
-    <value>200</value>
-    <description>to enable HADOOP:5784</description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.outofband.heartbeat</name>
-    <value>true</value>
-    <description>to enable MAPREDUCE:270</description>
-  </property>
-
-  <property>
-    <name>mapred.jobtracker.maxtasks.per.job</name>
-    <value>200000</value>
-    <final>true</final>
-    <description>The maximum number of tasks for a single job.
-      A value of -1 indicates that there is no maximum.  
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.kerberos.principal</name>
-    <value>jt/_HOST@${local.realm}</value>   
-    <description>
-       JT principal
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.kerberos.principal</name>
-    <value>tt/_HOST@${local.realm}</value>   
-    <description>       
-       TT principal.
-    </description>
-  </property>
-
-
-  <property>
-    <name>hadoop.job.history.user.location</name>
-    <value>none</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.keytab.file</name>
-    <value>/etc/security/keytabs/jt.service.keytab</value>
-    <description>
-        The keytab for the jobtracker principal.
-    </description>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.keytab.file</name>
-    <value>/etc/security/keytabs/tt.service.keytab</value>
-    <description>The filename of the keytab for the task tracker</description>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.staging.root.dir</name>
-    <value>/user</value>
-    <description>The Path prefix for where the staging directories should be 
-      placed. The next level is always the user's
-      name. It is a path in the default file system.
-    </description>
-  </property>
-
-
-  <property>
-    <name>mapreduce.job.acl-modify-job</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapreduce.job.acl-view-job</name>
-    <value>Dr.Who</value>
-  </property>
-
-  <property>
-    <name>mapreduce.tasktracker.group</name>
-    <value>${HADOOP_GROUP}</value>
-    <description>The group that the task controller uses for accessing the
-      task controller. The mapred user must be a member and users should *not*
-      be members.
-    </description> 
-  </property>
-
-  <property>
-    <name>mapred.acls.enabled</name>
-    <value>true</value>
-  </property>
-  
-  <property>
-    <name>mapred.jobtracker.taskScheduler</name>
-    <value>org.apache.hadoop.mapred.CapacityTaskScheduler</value>
-  </property>
-  <property>
-    <name>mapred.queue.names</name>
-    <value>default</value>
-  </property>
-
-  <!-- settings for the history server -->
-  <property>
-    <name>mapreduce.history.server.embedded</name>
-    <value>false</value>
-  </property>
-  <property>
-    <name>mapreduce.history.server.http.address</name>
-    <value>${HADOOP_JT_HOST}:51111</value>
-  </property>
-  <property>
-    <name>mapreduce.jobhistory.kerberos.principal</name>
-    <value>jt/_HOST@${local.realm}</value>   
-    <description>history server principal</description>
-  </property>
-  <property>
-    <name>mapreduce.jobhistory.keytab.file</name>
-    <value>/etc/security/keytabs/jt.service.keytab</value>
-    <description>
-        The keytab for the jobtracker principal.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.hosts</name>
-    <value>${HADOOP_CONF_DIR}/mapred.include</value>
-    <description>Names a file that contains the list of nodes that may
-    connect to the jobtracker.  If the value is empty, all hosts are
-    permitted.</description>
-  </property>
-          
-  <property>
-    <name>mapred.hosts.exclude</name>
-    <value>${HADOOP_CONF_DIR}/mapred.exclude</value>
-    <description>Names a file that contains the list of hosts that
-      should be excluded by the jobtracker.  If the value is empty, no
-      hosts are excluded.</description>
-  </property>
-  <property>
-    <name>mapred.jobtracker.retirejob.check</name>
-    <value>10000</value>
-  </property>
-  <property>
-    <name>mapred.jobtracker.retirejob.interval</name>
-    <value>0</value>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.mapmemory.mb</name>
-    <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.cluster.reducememory.mb</name>
-    <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.maxmapmemory.mb</name>
-    <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.jobtracker.maxreducememory.mb</name>
-    <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.map.memory.mb</name>
-    <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
-  </property>
-
-  <property>
-    <name>mapreduce.reduce.memory.mb</name>
-    <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
-  </property>
-</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/taskcontroller.cfg

@@ -1,21 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-# 
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-mapreduce.cluster.local.dir=${HADOOP_MAPRED_DIR}
-mapreduce.tasktracker.group=${HADOOP_GROUP}
-hadoop.log.dir=${HADOOP_LOG_DIR}/${HADOOP_MR_USER}

hadoop-common-project/hadoop-common/src/main/packages/update-hadoop-env.sh

@@ -1,172 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script configures hadoop-env.sh and symlinkis directories for 
-# relocating RPM locations.
-
-usage() {
-  echo "
-usage: $0 <parameters>
-  Required parameters:
-     --prefix=PREFIX             path to install into
-
-  Optional parameters:
-     --arch=i386                 OS Architecture
-     --bin-dir=PREFIX/bin        Executable directory
-     --conf-dir=/etc/hadoop      Configuration directory
-     --log-dir=/var/log/hadoop   Log directory
-     --pid-dir=/var/run          PID file location
-     --sbin-dir=PREFIX/sbin      System executable directory
-  "
-  exit 1
-}
-
-template_generator() {
-  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
-  cat $1 |
-  while read line ; do
-    while [[ "$line" =~ $REGEX ]] ; do
-      LHS=${BASH_REMATCH[1]}
-      RHS="$(eval echo "\"$LHS\"")"
-      line=${line//$LHS/$RHS}
-    done
-    echo $line >> $2
-  done
-}
-
-OPTS=$(getopt \
-  -n $0 \
-  -o '' \
-  -l 'arch:' \
-  -l 'prefix:' \
-  -l 'bin-dir:' \
-  -l 'conf-dir:' \
-  -l 'lib-dir:' \
-  -l 'log-dir:' \
-  -l 'pid-dir:' \
-  -l 'sbin-dir:' \
-  -l 'uninstall' \
-  -- "$@")
-
-if [ $? != 0 ] ; then
-    usage
-fi
-
-eval set -- "${OPTS}"
-while true ; do
-  case "$1" in
-    --arch)
-      ARCH=$2 ; shift 2
-      ;;
-    --prefix)
-      PREFIX=$2 ; shift 2
-      ;;
-    --bin-dir)
-      BIN_DIR=$2 ; shift 2
-      ;;
-    --log-dir)
-      LOG_DIR=$2 ; shift 2
-      ;;
-    --lib-dir)
-      LIB_DIR=$2 ; shift 2
-      ;;
-    --conf-dir)
-      CONF_DIR=$2 ; shift 2
-      ;;
-    --pid-dir)
-      PID_DIR=$2 ; shift 2
-      ;;
-    --sbin-dir)
-      SBIN_DIR=$2 ; shift 2
-      ;;
-    --uninstall)
-      UNINSTALL=1; shift
-      ;;
-    --)
-      shift ; break
-      ;;
-    *)
-      echo "Unknown option: $1"
-      usage
-      exit 1
-      ;;
-  esac
-done
-
-for var in PREFIX; do
-  if [ -z "$(eval "echo \$$var")" ]; then
-    echo Missing param: $var
-    usage
-  fi
-done
-
-ARCH=${ARCH:-i386}
-HADOOP_PREFIX=$PREFIX
-HADOOP_BIN_DIR=${BIN_DIR:-$PREFIX/bin}
-HADOOP_CONF_DIR=${CONF_DIR:-$PREFIX/etc/hadoop}
-HADOOP_LIB_DIR=${LIB_DIR:-$PREFIX/lib}
-HADOOP_LOG_DIR=${LOG_DIR:-$PREFIX/var/log}
-HADOOP_PID_DIR=${PID_DIR:-$PREFIX/var/run}
-HADOOP_SBIN_DIR=${SBIN_DIR:-$PREFIX/sbin}
-UNINSTALL=${UNINSTALL:-0}
-
-if [ "${ARCH}" != "i386" ]; then
-  HADOOP_LIB_DIR=${HADOOP_LIB_DIR}64
-fi
-
-if [ "${UNINSTALL}" -eq "1" ]; then
-  # Remove symlinks
-  if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
-    rm -rf ${HADOOP_PREFIX}/etc/hadoop
-  fi
-  rm -f /etc/default/hadoop-env.sh
-  if [ -d /etc/profile.d ]; then
-    rm -f /etc/profile.d/hadoop-env.sh
-  fi
-else
-  # Create symlinks
-  if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
-    mkdir -p ${HADOOP_PREFIX}/etc
-    ln -sf ${HADOOP_CONF_DIR} ${HADOOP_PREFIX}/etc/hadoop
-  fi
-  ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/default/hadoop-env.sh
-  if [ -d /etc/profile.d ]; then
-    ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh
-  fi
-
-  mkdir -p ${HADOOP_LOG_DIR}
-  chown root:hadoop ${HADOOP_LOG_DIR}
-  chmod 775 ${HADOOP_LOG_DIR}
-
-  if [ ! -d ${HADOOP_PID_DIR} ]; then
-    mkdir -p ${HADOOP_PID_DIR}
-    chown root:hadoop ${HADOOP_PID_DIR}
-    chmod 775 ${HADOOP_PID_DIR}
-  fi
-
-  TFILE="/tmp/$(basename $0).$$.tmp"
-  if [ -z "${JAVA_HOME}" ]; then
-    if [ -e /etc/debian_version ]; then
-      JAVA_HOME=`update-alternatives --config java | grep java | cut -f2 -d':' | cut -f2 -d' ' | sed -e 's/\/bin\/java//'`
-    else
-      JAVA_HOME=/usr/java/default
-    fi
-  fi
-  template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template $TFILE
-  cp ${TFILE} ${CONF_DIR}/hadoop-env.sh
-  rm -f ${TFILE}
-fi

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java

@@ -730,6 +730,55 @@ public class TestCodec {
     }
   }
 
+  @Test
+  public void testGzipLongOverflow() throws IOException {
+    LOG.info("testGzipLongOverflow");
+
+    // Don't use native libs for this test.
+    Configuration conf = new Configuration();
+    conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
+    assertFalse("ZlibFactory is using native libs against request",
+        ZlibFactory.isNativeZlibLoaded(conf));
+
+    // Ensure that the CodecPool has a BuiltInZlibInflater in it.
+    Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+    assertNotNull("zlibDecompressor is null!", zlibDecompressor);
+    assertTrue("ZlibFactory returned unexpected inflator",
+        zlibDecompressor instanceof BuiltInZlibInflater);
+    CodecPool.returnDecompressor(zlibDecompressor);
+
+    // Now create a GZip text file.
+    String tmpDir = System.getProperty("test.build.data", "/tmp/");
+    Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
+    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+      new GZIPOutputStream(new FileOutputStream(f.toString()))));
+
+    final int NBUF = 1024 * 4 + 1;
+    final char[] buf = new char[1024 * 1024];
+    for (int i = 0; i < buf.length; i++) buf[i] = '\0';
+    for (int i = 0; i < NBUF; i++) {
+      bw.write(buf);
+    }
+    bw.close();
+
+    // Now read it back, using the CodecPool to establish the
+    // decompressor to use.
+    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
+    CompressionCodec codec = ccf.getCodec(f);
+    Decompressor decompressor = CodecPool.getDecompressor(codec);
+    FileSystem fs = FileSystem.getLocal(conf);
+    InputStream is = fs.open(f);
+    is = codec.createInputStream(is, decompressor);
+    BufferedReader br = new BufferedReader(new InputStreamReader(is));
+    for (int j = 0; j < NBUF; j++) {
+      int n = br.read(buf);
+      assertEquals("got wrong read length!", n, buf.length);
+      for (int i = 0; i < buf.length; i++)
+        assertEquals("got wrong byte!", buf[i], '\0');
+    }
+    br.close();
+  }
+
   public void testGzipCodecWrite(boolean useNative) throws IOException {
     // Create a gzipped file using a compressor from the CodecPool,
     // and try to read it back via the regular GZIPInputStream.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestVLong.java

@@ -141,7 +141,7 @@ public class TestVLong extends TestCase {
       int shift = rng.nextInt(Long.SIZE) + 1;
       long mask = (1L << shift) - 1;
       long a = ((long) rng.nextInt()) << 32;
-      long b = ((long) rng.nextInt()) & 0xffffffff;
+      long b = ((long) rng.nextInt()) & 0xffffffffL;
       data[i] = (a + b) & mask;
     }
     

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java

@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends HttpServerFunctionalTest {
         "/jmx?get=java.lang:type=Memory::"));
     LOG.info("/jmx RESULT: "+result);
     assertReFind("\"ERROR\"", result);
+
+    // test to get JSONP result
+    result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
+    LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
+    assertReFind("^mycallback1\\(\\{", result);
+    assertReFind("\\}\\);$", result);
+
+    // negative test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback2\\(\\{", result);
+    assertReFind("\"ERROR\"", result);
+    assertReFind("\\}\\);$", result);
+
+    // test to get an attribute of a mbean as JSONP
+    result = readOutput(new URL(baseUrl,
+        "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
+    LOG.info("/jmx RESULT: "+result);
+    assertReFind("^mycallback3\\(\\{", result);
+    assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+    assertReFind("\"committed\"\\s*:", result);
+    assertReFind("\\}\\);$", result);
+
   }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java

@@ -18,9 +18,7 @@
 
 package org.apache.hadoop.metrics2.util;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -28,7 +26,6 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -54,18 +51,22 @@ public class TestSampleQuantiles {
     // Counts start off zero
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException from empty window");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    
+    // Snapshot should be null if there are no entries.
+    assertNull(estimator.snapshot());
 
     // Count increment correctly by 1
     estimator.insert(1337);
     assertEquals(estimator.getCount(), 1);
     estimator.snapshot();
     assertEquals(estimator.getSampleCount(), 1);
+    
+    assertEquals(
+        "50.00 %ile +/- 5.00%: 1337\n" +
+        "75.00 %ile +/- 2.50%: 1337\n" +
+        "90.00 %ile +/- 1.00%: 1337\n" +
+        "95.00 %ile +/- 0.50%: 1337\n" +
+        "99.00 %ile +/- 0.10%: 1337", estimator.toString());
   }
 
   /**
@@ -80,12 +81,7 @@ public class TestSampleQuantiles {
     estimator.clear();
     assertEquals(estimator.getCount(), 0);
     assertEquals(estimator.getSampleCount(), 0);
-    try {
-      estimator.snapshot();
-      fail("Expected IOException for an empty window.");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("No samples", e);
-    }
+    assertNull(estimator.snapshot());
   }
 
   /**

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java

@@ -142,16 +142,14 @@ public class TestPureJavaCrc32 {
       for (int j = 0; j < tables.length; j++) {
         final int[] t = tables[j];
         final StringBuilder b = new StringBuilder();
-        b.append(String.format("  static final int[] " + nameformat
-            + " = new int[] {", j));
+        b.append(String.format("    /* "+ nameformat +" */", j));
         for (int i = 0; i < t.length;) {
           b.append("\n    ");
           for(int k = 0; k < 4; k++) {
             b.append(String.format("0x%08X, ", t[i++]));
           }
         }
-        b.setCharAt(b.length() - 2, '\n');
-        s[j] = b.toString() + " };\n";
+        s[j] = b.toString();
       }
       return s;
     }
@@ -159,10 +157,23 @@ public class TestPureJavaCrc32 {
     @Override
     public String toString() {
       final StringBuilder b = new StringBuilder();
-      for(String s : toStrings(String.format("T%d_",
-          Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
+
+      final String tableFormat = String.format("T%d_", 
+        Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
+      final String startFormat = "  private static final int "+tableFormat+"_start = %d*256;";
+
+      for (int j = 0; j < tables.length; j++) {
+        b.append(String.format(startFormat, j, j));
+        b.append("\n");
+      }
+
+      b.append("  private static final int[] T = new int[] {");
+      for(String s : toStrings(tableFormat)) {
+        b.append("\n");
         b.append(s);
       }
+      b.setCharAt(b.length() - 2, '\n');
+      b.append(" };\n");
       return b.toString();
     }
 

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -147,6 +147,9 @@ Trunk (Unreleased)
     Block Pool Used, Block Pool Used(%) and Failed Volumes.
     (Brahma Reddy Battula via suresh)
 
+    HDFS-4052. BlockManager#invalidateWork should print log outside the lock.
+    (Jing Zhao via suresh)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -224,8 +227,6 @@ Trunk (Unreleased)
     HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from
     Command. (Jing Zhao via suresh)
 
-    HDFS-3678. Edit log files are never being purged from 2NN. (atm)
-
     HADOOP-8158. Interrupting hadoop fs -put from the command line
     causes a LeaseExpiredException. (daryn via harsh)
 
@@ -343,6 +344,8 @@ Release 2.0.3-alpha - Unreleased
     HDFS-3912. Detect and avoid stale datanodes for writes.
     (Jing Zhao via suresh)
 
+    HDFS-4059. Add number of stale DataNodes to metrics. (Jing Zhao via suresh)
+
   IMPROVEMENTS
   
     HDFS-3925. Prettify PipelineAck#toString() for printing to a log
@@ -388,6 +391,20 @@ Release 2.0.3-alpha - Unreleased
     HDFS-4036. Remove "throw UnresolvedLinkException" from
     FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo)
 
+    HDFS-2946. HA: Put a cap on the number of completed edits files retained
+    by the NN. (atm)
+
+    HDFS-4029. GenerationStamp should use an AtomicLong. (eli)
+
+    HDFS-4068. DatanodeID and DatanodeInfo member should be private. (eli)
+
+    HDFS-4073. Two minor improvements to FSDirectory.  (Jing Zhao via szetszwo)
+
+    HDFS-4074. Remove the unused default constructor from INode.  (Brandon Li
+    via szetszwo)
+
+    HDFS-4053. Increase the default block size. (eli)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -440,6 +457,26 @@ Release 2.0.3-alpha - Unreleased
     HDFS-4044. Duplicate ChecksumType definition in HDFS .proto files.
     (Binglin Chang via suresh)
 
+    HDFS-4049. Fix hflush performance regression due to nagling delays
+    (todd)
+
+    HDFS-3678. Edit log files are never being purged from 2NN. (atm)
+
+    HDFS-4058. DirectoryScanner may fail with IOOB if the directory
+    scanning threads return out of volume order. (eli)
+
+    HDFS-3985. Add timeouts to TestMulitipleNNDataBlockScanner. (todd via eli)
+
+    HDFS-4061. TestBalancer and TestUnderReplicatedBlocks need timeouts. (eli)
+
+    HDFS-3997. OfflineImageViewer incorrectly passes value of imageVersion when
+    visiting IS_COMPRESSED element. (Mithun Radhakrishnan via atm)
+
+    HDFS-4055. TestAuditLogs is flaky. (Binglin Chang via eli)
+
+    HDFS-4072. On file deletion remove corresponding blocks pending
+    replications. (Jing Zhao via suresh)
+
 Release 2.0.2-alpha - 2012-09-07 
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
 public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_BLOCK_SIZE_KEY = "dfs.blocksize";
-  public static final long    DFS_BLOCK_SIZE_DEFAULT = 64*1024*1024;
+  public static final long    DFS_BLOCK_SIZE_DEFAULT = 128*1024*1024;
   public static final String  DFS_REPLICATION_KEY = "dfs.replication";
   public static final short   DFS_REPLICATION_DEFAULT = 3;
   public static final String  DFS_STREAM_BUFFER_SIZE_KEY = "dfs.stream-buffer-size";
@@ -162,6 +162,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT = 2;
   public static final String  DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY = "dfs.namenode.num.extra.edits.retained";
   public static final int     DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT = 1000000; //1M
+  public static final String  DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY = "dfs.namenode.max.extra.edits.segments.retained";
+  public static final int     DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_DEFAULT = 10000; // 10k
   public static final String  DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY = "dfs.namenode.min.supported.datanode.version";
   public static final String  DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT = "3.0.0-SNAPSHOT";
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java

@@ -37,12 +37,12 @@ import org.apache.hadoop.classification.InterfaceStability;
 public class DatanodeID implements Comparable<DatanodeID> {
   public static final DatanodeID[] EMPTY_ARRAY = {};
 
-  protected String ipAddr;     // IP address
-  protected String hostName;   // hostname
-  protected String storageID;  // unique per cluster storageID
-  protected int xferPort;      // data streaming port
-  protected int infoPort;      // info server port
-  protected int ipcPort;       // IPC server port
+  private String ipAddr;     // IP address
+  private String hostName;   // hostname
+  private String storageID;  // unique per cluster storageID
+  private int xferPort;      // data streaming port
+  private int infoPort;      // info server port
+  private int ipcPort;       // IPC server port
 
   public DatanodeID(DatanodeID from) {
     this(from.getIpAddr(),

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java

@@ -37,13 +37,13 @@ import org.apache.hadoop.util.Time;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class DatanodeInfo extends DatanodeID implements Node {
-  protected long capacity;
-  protected long dfsUsed;
-  protected long remaining;
-  protected long blockPoolUsed;
-  protected long lastUpdate;
-  protected int xceiverCount;
-  protected String location = NetworkTopology.DEFAULT_RACK;
+  private long capacity;
+  private long dfsUsed;
+  private long remaining;
+  private long blockPoolUsed;
+  private long lastUpdate;
+  private int xceiverCount;
+  private String location = NetworkTopology.DEFAULT_RACK;
   
   // Datanode administrative states
   public enum AdminStates {
@@ -81,8 +81,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
     this.lastUpdate = from.getLastUpdate();
     this.xceiverCount = from.getXceiverCount();
     this.location = from.getNetworkLocation();
-    this.adminState = from.adminState;
-    this.hostName = from.hostName;
+    this.adminState = from.getAdminState();
   }
 
   public DatanodeInfo(DatanodeID nodeID) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java

@@ -53,14 +53,8 @@ public class PacketReceiver implements Closeable {
   private final boolean useDirectBuffers;
 
   /**
-   * Internal buffer for reading the length prefixes at the start of
-   * the packet.
-   */
-  private final ByteBuffer lengthPrefixBuf = ByteBuffer.allocate(
-      PacketHeader.PKT_LENGTHS_LEN);
-
-  /**
-   * The entirety of the most recently read packet, excepting the
+   * The entirety of the most recently read packet.
+   * The first PKT_LENGTHS_LEN bytes of this buffer are the
    * length prefixes.
    */
   private ByteBuffer curPacketBuf = null;
@@ -82,6 +76,7 @@ public class PacketReceiver implements Closeable {
   
   public PacketReceiver(boolean useDirectBuffers) {
     this.useDirectBuffers = useDirectBuffers;
+    reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN);
   }
 
   public PacketHeader getHeader() {
@@ -133,11 +128,12 @@ public class PacketReceiver implements Closeable {
     //            checksums were not requested
     // DATA       the actual block data
     Preconditions.checkState(curHeader == null || !curHeader.isLastPacketInBlock());
-    
-    lengthPrefixBuf.clear();
-    doReadFully(ch, in, lengthPrefixBuf);
-    lengthPrefixBuf.flip();
-    int payloadLen = lengthPrefixBuf.getInt();
+
+    curPacketBuf.clear();
+    curPacketBuf.limit(PacketHeader.PKT_LENGTHS_LEN);
+    doReadFully(ch, in, curPacketBuf);
+    curPacketBuf.flip();
+    int payloadLen = curPacketBuf.getInt();
     
     if (payloadLen < Ints.BYTES) {
       // The "payload length" includes its own length. Therefore it
@@ -146,7 +142,7 @@ public class PacketReceiver implements Closeable {
           payloadLen);
     }
     int dataPlusChecksumLen = payloadLen - Ints.BYTES;
-    int headerLen = lengthPrefixBuf.getShort();
+    int headerLen = curPacketBuf.getShort();
     if (headerLen < 0) {
       throw new IOException("Invalid header length " + headerLen);
     }
@@ -166,13 +162,17 @@ public class PacketReceiver implements Closeable {
 
     // Make sure we have space for the whole packet, and
     // read it.
-    reallocPacketBuf(dataPlusChecksumLen + headerLen);
+    reallocPacketBuf(PacketHeader.PKT_LENGTHS_LEN +
+        dataPlusChecksumLen + headerLen);
     curPacketBuf.clear();
-    curPacketBuf.limit(dataPlusChecksumLen + headerLen);
+    curPacketBuf.position(PacketHeader.PKT_LENGTHS_LEN);
+    curPacketBuf.limit(PacketHeader.PKT_LENGTHS_LEN +
+        dataPlusChecksumLen + headerLen);
     doReadFully(ch, in, curPacketBuf);
     curPacketBuf.flip();
+    curPacketBuf.position(PacketHeader.PKT_LENGTHS_LEN);
 
-    // Extract the header from the front of the buffer.
+    // Extract the header from the front of the buffer (after the length prefixes)
     byte[] headerBuf = new byte[headerLen];
     curPacketBuf.get(headerBuf);
     if (curHeader == null) {
@@ -197,10 +197,6 @@ public class PacketReceiver implements Closeable {
   public void mirrorPacketTo(DataOutputStream mirrorOut) throws IOException {
     Preconditions.checkState(!useDirectBuffers,
         "Currently only supported for non-direct buffers");
-    assert lengthPrefixBuf.capacity() == PacketHeader.PKT_LENGTHS_LEN;
-    mirrorOut.write(lengthPrefixBuf.array(),
-        lengthPrefixBuf.arrayOffset(),
-        lengthPrefixBuf.capacity());
     mirrorOut.write(curPacketBuf.array(),
         curPacketBuf.arrayOffset(),
         curPacketBuf.remaining());
@@ -223,23 +219,36 @@ public class PacketReceiver implements Closeable {
 
   private void reslicePacket(
       int headerLen, int checksumsLen, int dataLen) {
+    // Packet structure (refer to doRead() for details):
+    //   PLEN    HLEN      HEADER     CHECKSUMS  DATA
+    //   32-bit  16-bit   <protobuf>  <variable length>
+    //   |--- lenThroughHeader ----|
+    //   |----------- lenThroughChecksums   ----|
+    //   |------------------- lenThroughData    ------| 
+    int lenThroughHeader = PacketHeader.PKT_LENGTHS_LEN + headerLen;
+    int lenThroughChecksums = lenThroughHeader + checksumsLen;
+    int lenThroughData = lenThroughChecksums + dataLen;
+
     assert dataLen >= 0 : "invalid datalen: " + dataLen;
-    
-    assert curPacketBuf.position() == headerLen;
-    assert checksumsLen + dataLen == curPacketBuf.remaining() :
+    assert curPacketBuf.position() == lenThroughHeader;
+    assert curPacketBuf.limit() == lenThroughData :
       "headerLen= " + headerLen + " clen=" + checksumsLen + " dlen=" + dataLen +
       " rem=" + curPacketBuf.remaining();
-    
-    curPacketBuf.position(headerLen);
-    curPacketBuf.limit(headerLen + checksumsLen);
+
+    // Slice the checksums.
+    curPacketBuf.position(lenThroughHeader);
+    curPacketBuf.limit(lenThroughChecksums);
     curChecksumSlice = curPacketBuf.slice();
 
-    curPacketBuf.position(headerLen + checksumsLen);
-    curPacketBuf.limit(headerLen + checksumsLen + dataLen);
+    // Slice the data.
+    curPacketBuf.position(lenThroughChecksums);
+    curPacketBuf.limit(lenThroughData);
     curDataSlice = curPacketBuf.slice();
     
+    // Reset buffer to point to the entirety of the packet (including
+    // length prefixes)
     curPacketBuf.position(0);
-    curPacketBuf.limit(headerLen + checksumsLen + dataLen);
+    curPacketBuf.limit(lenThroughData);
   }
 
   
@@ -258,12 +267,21 @@ public class PacketReceiver implements Closeable {
     // one.
     if (curPacketBuf == null ||
         curPacketBuf.capacity() < atLeastCapacity) {
-      returnPacketBufToPool();
+      ByteBuffer newBuf;
       if (useDirectBuffers) {
-        curPacketBuf = bufferPool.getBuffer(atLeastCapacity);
+        newBuf = bufferPool.getBuffer(atLeastCapacity);
       } else {
-        curPacketBuf = ByteBuffer.allocate(atLeastCapacity);
+        newBuf = ByteBuffer.allocate(atLeastCapacity);
       }
+      // If reallocing an existing buffer, copy the old packet length
+      // prefixes over
+      if (curPacketBuf != null) {
+        curPacketBuf.flip();
+        newBuf.put(curPacketBuf);
+      }
+      
+      returnPacketBufToPool();
+      curPacketBuf = newBuf;
     }
   }
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -288,7 +288,7 @@ public class BlockManager {
   }
 
   private static BlockTokenSecretManager createBlockTokenSecretManager(
-      final Configuration conf) throws IOException {
+      final Configuration conf) {
     final boolean isEnabled = conf.getBoolean(
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
         DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@@ -1260,7 +1260,7 @@ public class BlockManager {
           // Move the block-replication into a "pending" state.
           // The reason we use 'pending' is so we can retry
           // replications that fail after an appropriate amount of time.
-          pendingReplications.add(block, targets.length);
+          pendingReplications.increment(block, targets.length);
           if(NameNode.stateChangeLog.isDebugEnabled()) {
             NameNode.stateChangeLog.debug(
                 "BLOCK* block " + block
@@ -1306,8 +1306,11 @@ public class BlockManager {
 
   /**
    * Choose target datanodes according to the replication policy.
-   * @throws IOException if the number of targets < minimum replication.
-   * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, HashMap, long)
+   * 
+   * @throws IOException
+   *           if the number of targets < minimum replication.
+   * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor,
+   *      List, boolean, HashMap, long)
    */
   public DatanodeDescriptor[] chooseTarget(final String src,
       final int numOfReplicas, final DatanodeDescriptor client,
@@ -1811,7 +1814,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
 
   /**
    * Queue the given reported block for later processing in the
-   * standby node. {@see PendingDataNodeMessages}.
+   * standby node. @see PendingDataNodeMessages.
    * @param reason a textual reason to report in the debug logs
    */
   private void queueReportedBlock(DatanodeDescriptor dn, Block block,
@@ -1976,14 +1979,15 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
   }
   
   /**
-   * Faster version of {@link addStoredBlock()}, intended for use with 
-   * initial block report at startup.  If not in startup safe mode, will
-   * call standard addStoredBlock().
-   * Assumes this method is called "immediately" so there is no need to
-   * refresh the storedBlock from blocksMap.
-   * Doesn't handle underReplication/overReplication, or worry about
+   * Faster version of
+   * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)}
+   * , intended for use with initial block report at startup. If not in startup
+   * safe mode, will call standard addStoredBlock(). Assumes this method is
+   * called "immediately" so there is no need to refresh the storedBlock from
+   * blocksMap. Doesn't handle underReplication/overReplication, or worry about
    * pendingReplications or corruptReplicas, because it's in startup safe mode.
    * Doesn't log every block, because there are typically millions of them.
+   * 
    * @throws IOException
    */
   private void addStoredBlockImmediate(BlockInfo storedBlock,
@@ -2505,7 +2509,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     //
     // Modify the blocks->datanode map and node's map.
     //
-    pendingReplications.remove(block);
+    pendingReplications.decrement(block);
     processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED,
         delHintNode);
   }
@@ -2641,7 +2645,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
   }
 
   /** 
-   * Simpler, faster form of {@link countNodes()} that only returns the number
+   * Simpler, faster form of {@link #countNodes(Block)} that only returns the number
    * of live nodes.  If in startup safemode (or its 30-sec extension period),
    * then it gains speed by ignoring issues of excess replicas or nodes
    * that are decommissioned or in process of becoming decommissioned.
@@ -2790,6 +2794,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     addToInvalidates(block);
     corruptReplicas.removeFromCorruptReplicasMap(block);
     blocksMap.removeBlock(block);
+    // Remove the block from pendingReplications
+    pendingReplications.remove(block);
     if (postponedMisreplicatedBlocks.remove(block)) {
       postponedMisreplicatedBlocksCount--;
     }
@@ -2856,6 +2862,9 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
    * @return number of blocks scheduled for removal during this iteration.
    */
   private int invalidateWorkForOneNode(String nodeId) {
+    final List<Block> toInvalidate;
+    final DatanodeDescriptor dn;
+    
     namesystem.writeLock();
     try {
       // blocks should not be replicated or removed if safe mode is on
@@ -2865,10 +2874,23 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
       }
       // get blocks to invalidate for the nodeId
       assert nodeId != null;
-      return invalidateBlocks.invalidateWork(nodeId);
+      dn = datanodeManager.getDatanode(nodeId);
+      if (dn == null) {
+        invalidateBlocks.remove(nodeId);
+        return 0;
+      }
+      toInvalidate = invalidateBlocks.invalidateWork(nodeId, dn);
+      if (toInvalidate == null) {
+        return 0;
+      }
     } finally {
       namesystem.writeUnlock();
     }
+    if (NameNode.stateChangeLog.isInfoEnabled()) {
+      NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName()
+          + ": ask " + dn + " to delete " + toInvalidate);
+    }
+    return toInvalidate.size();
   }
 
   boolean blockHasEnoughRacks(Block b) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java

@@ -276,11 +276,11 @@ public class DatanodeDescriptor extends DatanodeInfo {
   }
 
   public void resetBlocks() {
-    this.capacity = 0;
-    this.remaining = 0;
-    this.blockPoolUsed = 0;
-    this.dfsUsed = 0;
-    this.xceiverCount = 0;
+    setCapacity(0);
+    setRemaining(0);
+    setBlockPoolUsed(0);
+    setDfsUsed(0);
+    setXceiverCount(0);
     this.blockList = null;
     this.invalidateBlocks.clear();
     this.volumeFailures = 0;
@@ -303,15 +303,15 @@ public class DatanodeDescriptor extends DatanodeInfo {
    */
   public void updateHeartbeat(long capacity, long dfsUsed, long remaining,
       long blockPoolUsed, int xceiverCount, int volFailures) {
-    this.capacity = capacity;
-    this.dfsUsed = dfsUsed;
-    this.remaining = remaining;
-    this.blockPoolUsed = blockPoolUsed;
-    this.lastUpdate = Time.now();
-    this.xceiverCount = xceiverCount;
+    setCapacity(capacity);
+    setRemaining(remaining);
+    setBlockPoolUsed(blockPoolUsed);
+    setDfsUsed(dfsUsed);
+    setXceiverCount(xceiverCount);
+    setLastUpdate(Time.now());    
     this.volumeFailures = volFailures;
     this.heartbeatedSinceFailover = true;
-    rollBlocksScheduled(lastUpdate);
+    rollBlocksScheduled(getLastUpdate());
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java

@@ -567,7 +567,7 @@ public class DatanodeManager {
   /**
    * Decommission the node if it is in exclude list.
    */
-  private void checkDecommissioning(DatanodeDescriptor nodeReg, String ipAddr) { 
+  private void checkDecommissioning(DatanodeDescriptor nodeReg) { 
     // If the registered node is in exclude list, then decommission it
     if (inExcludedHostsList(nodeReg)) {
       startDecommission(nodeReg);
@@ -713,7 +713,7 @@ public class DatanodeManager {
         
       // also treat the registration message as a heartbeat
       heartbeatManager.register(nodeS);
-      checkDecommissioning(nodeS, dnAddress);
+      checkDecommissioning(nodeS);
       return;
     } 
 
@@ -733,7 +733,7 @@ public class DatanodeManager {
       = new DatanodeDescriptor(nodeReg, NetworkTopology.DEFAULT_RACK);
     resolveNetworkLocation(nodeDescr);
     addDatanode(nodeDescr);
-    checkDecommissioning(nodeDescr, dnAddress);
+    checkDecommissioning(nodeDescr);
     
     // also treat the registration message as a heartbeat
     // no need to update its timestamp
@@ -885,7 +885,7 @@ public class DatanodeManager {
    * @return Return the current number of stale DataNodes (detected by
    * HeartbeatManager). 
    */
-  int getNumStaleNodes() {
+  public int getNumStaleNodes() {
     return this.numStaleNodes;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java

@@ -134,26 +134,7 @@ class InvalidateBlocks {
     return new ArrayList<String>(node2blocks.keySet());
   }
 
-  /** Invalidate work for the storage. */
-  int invalidateWork(final String storageId) {
-    final DatanodeDescriptor dn = datanodeManager.getDatanode(storageId);
-    if (dn == null) {
-      remove(storageId);
-      return 0;
-    }
-    final List<Block> toInvalidate = invalidateWork(storageId, dn);
-    if (toInvalidate == null) {
-      return 0;
-    }
-
-    if (NameNode.stateChangeLog.isInfoEnabled()) {
-      NameNode.stateChangeLog.info("BLOCK* " + getClass().getSimpleName()
-          + ": ask " + dn + " to delete " + toInvalidate);
-    }
-    return toInvalidate.size();
-  }
-
-  private synchronized List<Block> invalidateWork(
+  synchronized List<Block> invalidateWork(
       final String storageId, final DatanodeDescriptor dn) {
     final LightWeightHashSet<Block> set = node2blocks.get(storageId);
     if (set == null) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java

@@ -72,7 +72,7 @@ class PendingReplicationBlocks {
   /**
    * Add a block to the list of pending Replications
    */
-  void add(Block block, int numReplicas) {
+  void increment(Block block, int numReplicas) {
     synchronized (pendingReplications) {
       PendingBlockInfo found = pendingReplications.get(block);
       if (found == null) {
@@ -89,7 +89,7 @@ class PendingReplicationBlocks {
    * Decrement the number of pending replication requests
    * for this block.
    */
-  void remove(Block block) {
+  void decrement(Block block) {
     synchronized (pendingReplications) {
       PendingBlockInfo found = pendingReplications.get(block);
       if (found != null) {
@@ -104,6 +104,16 @@ class PendingReplicationBlocks {
     }
   }
 
+  /**
+   * Remove the record about the given block from pendingReplications.
+   * @param block The given block whose pending replication requests need to be
+   *              removed
+   */
+  void remove(Block block) {
+    synchronized (pendingReplications) {
+      pendingReplications.remove(block);
+    }
+  }
 
   public void clear() {
     synchronized (pendingReplications) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.common;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 
 /****************************************************************
@@ -35,7 +37,7 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
    */
   public static final long GRANDFATHER_GENERATION_STAMP = 0;
 
-  private volatile long genstamp;
+  private AtomicLong genstamp = new AtomicLong();
 
   /**
    * Create a new instance, initialized to FIRST_VALID_STAMP.
@@ -48,35 +50,36 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
    * Create a new instance, initialized to the specified value.
    */
   GenerationStamp(long stamp) {
-    this.genstamp = stamp;
+    genstamp.set(stamp);
   }
 
   /**
    * Returns the current generation stamp
    */
   public long getStamp() {
-    return this.genstamp;
+    return genstamp.get();
   }
 
   /**
    * Sets the current generation stamp
    */
   public void setStamp(long stamp) {
-    this.genstamp = stamp;
+    genstamp.set(stamp);
   }
 
   /**
    * First increments the counter and then returns the stamp 
    */
-  public synchronized long nextStamp() {
-    this.genstamp++;
-    return this.genstamp;
+  public long nextStamp() {
+    return genstamp.incrementAndGet();
   }
 
   @Override // Comparable
   public int compareTo(GenerationStamp that) {
-    return this.genstamp < that.genstamp ? -1 :
-           this.genstamp > that.genstamp ? 1 : 0;
+    long stamp1 = this.genstamp.get();
+    long stamp2 = that.genstamp.get();
+    return stamp1 < stamp2 ? -1 :
+           stamp1 > stamp2 ? 1 : 0;
   }
 
   @Override // Object
@@ -89,6 +92,7 @@ public class GenerationStamp implements Comparable<GenerationStamp> {
 
   @Override // Object
   public int hashCode() {
-    return (int) (genstamp^(genstamp>>>32));
+    long stamp = genstamp.get();
+    return (int) (stamp^(stamp>>>32));
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java

@@ -431,16 +431,16 @@ public class DirectoryScanner implements Runnable {
   private Map<String, ScanInfo[]> getDiskReport() {
     // First get list of data directories
     final List<? extends FsVolumeSpi> volumes = dataset.getVolumes();
-    ArrayList<ScanInfoPerBlockPool> dirReports =
-      new ArrayList<ScanInfoPerBlockPool>(volumes.size());
-    
+
+    // Use an array since the threads may return out of order and
+    // compilersInProgress#keySet may return out of order as well.
+    ScanInfoPerBlockPool[] dirReports = new ScanInfoPerBlockPool[volumes.size()];
+
     Map<Integer, Future<ScanInfoPerBlockPool>> compilersInProgress =
       new HashMap<Integer, Future<ScanInfoPerBlockPool>>();
+
     for (int i = 0; i < volumes.size(); i++) {
-      if (!isValid(dataset, volumes.get(i))) {
-        // volume is invalid
-        dirReports.add(i, null);
-      } else {
+      if (isValid(dataset, volumes.get(i))) {
         ReportCompiler reportCompiler =
           new ReportCompiler(volumes.get(i));
         Future<ScanInfoPerBlockPool> result = 
@@ -452,7 +452,7 @@ public class DirectoryScanner implements Runnable {
     for (Entry<Integer, Future<ScanInfoPerBlockPool>> report :
         compilersInProgress.entrySet()) {
       try {
-        dirReports.add(report.getKey(), report.getValue().get());
+        dirReports[report.getKey()] = report.getValue().get();
       } catch (Exception ex) {
         LOG.error("Error compiling report", ex);
         // Propagate ex to DataBlockScanner to deal with
@@ -465,7 +465,7 @@ public class DirectoryScanner implements Runnable {
     for (int i = 0; i < volumes.size(); i++) {
       if (isValid(dataset, volumes.get(i))) {
         // volume is still valid
-        list.addAll(dirReports.get(i));
+        list.addAll(dirReports[i]);
       }
     }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java

@@ -290,13 +290,18 @@ public class FSDirectory implements Closeable {
     try {
       newNode = addNode(path, newNode, UNKNOWN_DISK_SPACE);
     } catch (IOException e) {
+      if(NameNode.stateChangeLog.isDebugEnabled()) {
+        NameNode.stateChangeLog.debug(
+            "DIR* FSDirectory.unprotectedAddFile: exception when add " + path
+                + " to the file system", e);
+      }
       return null;
     }
     return newNode;
   }
 
   INodeDirectory addToParent(byte[] src, INodeDirectory parentINode,
-      INode newNode, boolean propagateModTime) throws UnresolvedLinkException {
+      INode newNode, boolean propagateModTime) {
     // NOTE: This does not update space counts for parents
     INodeDirectory newParent = null;
     writeLock();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

@@ -1174,6 +1174,11 @@ public class FSEditLog implements LogsPurgeable {
       // TODO: are we sure this is OK?
     }
   }
+  
+  public void selectInputStreams(Collection<EditLogInputStream> streams,
+      long fromTxId, boolean inProgressOk) {
+    journalSet.selectInputStreams(streams, fromTxId, inProgressOk);
+  }
 
   public Collection<EditLogInputStream> selectInputStreams(
       long fromTxId, long toAtLeastTxId) throws IOException {
@@ -1191,7 +1196,7 @@ public class FSEditLog implements LogsPurgeable {
       long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery,
       boolean inProgressOk) throws IOException {
     List<EditLogInputStream> streams = new ArrayList<EditLogInputStream>();
-    journalSet.selectInputStreams(streams, fromTxId, inProgressOk);
+    selectInputStreams(streams, fromTxId, inProgressOk);
 
     try {
       checkForGaps(streams, fromTxId, toAtLeastTxId, inProgressOk);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -4677,6 +4677,13 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
   public int getNumDeadDataNodes() {
     return getBlockManager().getDatanodeManager().getNumDeadDataNodes();
   }
+  
+  @Override // FSNamesystemMBean
+  @Metric({"StaleDataNodes", 
+    "Number of datanodes marked stale due to delayed heartbeat"})
+  public int getNumStaleDataNodes() {
+    return getBlockManager().getDatanodeManager().getNumStaleNodes();
+  }
 
   /**
    * Sets the generation stamp for this filesystem

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java

@@ -247,6 +247,11 @@ public class FileJournalManager implements JournalManager {
     LOG.debug(this + ": selecting input streams starting at " + fromTxId + 
         (inProgressOk ? " (inProgress ok) " : " (excluding inProgress) ") +
         "from among " + elfs.size() + " candidate file(s)");
+    addStreamsToCollectionFromFiles(elfs, streams, fromTxId, inProgressOk);
+  }
+  
+  static void addStreamsToCollectionFromFiles(Collection<EditLogFile> elfs,
+      Collection<EditLogInputStream> streams, long fromTxId, boolean inProgressOk) {
     for (EditLogFile elf : elfs) {
       if (elf.isInProgress()) {
         if (!inProgressOk) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java

@@ -96,13 +96,6 @@ abstract class INode implements Comparable<byte[]> {
     }
   }
 
-  protected INode() {
-    name = null;
-    parent = null;
-    modificationTime = 0;
-    accessTime = 0;
-  }
-
   INode(PermissionStatus permissions, long mTime, long atime) {
     this.name = null;
     this.parent = null;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java

@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collection;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -56,21 +55,6 @@ public interface JournalManager extends Closeable, FormatConfirmable,
    */
   void finalizeLogSegment(long firstTxId, long lastTxId) throws IOException;
 
-   /**
-   * Get a list of edit log input streams.  The list will start with the
-   * stream that contains fromTxnId, and continue until the end of the journal
-   * being managed.
-   * 
-   * @param fromTxnId the first transaction id we want to read
-   * @param inProgressOk whether or not in-progress streams should be returned
-   *
-   * @return a list of streams
-   * @throws IOException if the underlying storage has an error or is otherwise
-   * inaccessible
-   */
-  void selectInputStreams(Collection<EditLogInputStream> streams,
-      long fromTxnId, boolean inProgressOk) throws IOException;
-
   /**
    * Set the amount of memory that this stream should use to buffer edits
    */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
+import java.util.Collection;
 
 /**
  * Interface used to abstract over classes which manage edit logs that may need
@@ -33,5 +34,20 @@ interface LogsPurgeable {
    * @throws IOException in the event of error
    */
   public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException;
-
+  
+  /**
+   * Get a list of edit log input streams.  The list will start with the
+   * stream that contains fromTxnId, and continue until the end of the journal
+   * being managed.
+   * 
+   * @param fromTxId the first transaction id we want to read
+   * @param inProgressOk whether or not in-progress streams should be returned
+   *
+   * @return a list of streams
+   * @throws IOException if the underlying storage has an error or is otherwise
+   * inaccessible
+   */
+  void selectInputStreams(Collection<EditLogInputStream> streams,
+      long fromTxId, boolean inProgressOk) throws IOException;
+  
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java

@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.TreeSet;
 
@@ -32,6 +34,7 @@ import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
@@ -48,6 +51,7 @@ public class NNStorageRetentionManager {
   
   private final int numCheckpointsToRetain;
   private final long numExtraEditsToRetain;
+  private final int maxExtraEditsSegmentsToRetain;
   private static final Log LOG = LogFactory.getLog(
       NNStorageRetentionManager.class);
   private final NNStorage storage;
@@ -65,6 +69,9 @@ public class NNStorageRetentionManager {
     this.numExtraEditsToRetain = conf.getLong(
         DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,
         DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_DEFAULT);
+    this.maxExtraEditsSegmentsToRetain = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_KEY,
+        DFSConfigKeys.DFS_NAMENODE_MAX_EXTRA_EDITS_SEGMENTS_RETAINED_DEFAULT);
     Preconditions.checkArgument(numCheckpointsToRetain > 0,
         "Must retain at least one checkpoint");
     Preconditions.checkArgument(numExtraEditsToRetain >= 0,
@@ -94,7 +101,39 @@ public class NNStorageRetentionManager {
     // provide a "cushion" of older txns that we keep, which is
     // handy for HA, where a remote node may not have as many
     // new images.
-    long purgeLogsFrom = Math.max(0, minImageTxId + 1 - numExtraEditsToRetain);
+    //
+    // First, determine the target number of extra transactions to retain based
+    // on the configured amount.
+    long minimumRequiredTxId = minImageTxId + 1;
+    long purgeLogsFrom = Math.max(0, minimumRequiredTxId - numExtraEditsToRetain);
+    
+    ArrayList<EditLogInputStream> editLogs = new ArrayList<EditLogInputStream>();
+    purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false);
+    Collections.sort(editLogs, new Comparator<EditLogInputStream>() {
+      @Override
+      public int compare(EditLogInputStream a, EditLogInputStream b) {
+        return ComparisonChain.start()
+            .compare(a.getFirstTxId(), b.getFirstTxId())
+            .compare(a.getLastTxId(), b.getLastTxId())
+            .result();
+      }
+    });
+    
+    // Next, adjust the number of transactions to retain if doing so would mean
+    // keeping too many segments around.
+    while (editLogs.size() > maxExtraEditsSegmentsToRetain) {
+      purgeLogsFrom = editLogs.get(0).getFirstTxId();
+      editLogs.remove(0);
+    }
+    
+    // Finally, ensure that we're not trying to purge any transactions that we
+    // actually need.
+    if (purgeLogsFrom > minimumRequiredTxId) {
+      throw new AssertionError("Should not purge more edits than required to "
+          + "restore: " + purgeLogsFrom + " should be <= "
+          + minimumRequiredTxId);
+    }
+    
     purgeableLogs.purgeLogsOlderThan(purgeLogsFrom);
   }
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java

@@ -751,6 +751,24 @@ public class SecondaryNameNode implements Runnable {
           }
         }
       }
+
+      @Override
+      public void selectInputStreams(Collection<EditLogInputStream> streams,
+          long fromTxId, boolean inProgressOk) {
+        Iterator<StorageDirectory> iter = storage.dirIterator();
+        while (iter.hasNext()) {
+          StorageDirectory dir = iter.next();
+          List<EditLogFile> editFiles;
+          try {
+            editFiles = FileJournalManager.matchEditLogs(
+                dir.getCurrentDir());
+          } catch (IOException ioe) {
+            throw new RuntimeException(ioe);
+          }
+          FileJournalManager.addStreamsToCollectionFromFiles(editFiles, streams,
+              fromTxId, inProgressOk);
+        }
+      }
       
     }
     

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java

@@ -112,4 +112,10 @@ public interface FSNamesystemMBean {
    * @return number of dead data nodes
    */
   public int getNumDeadDataNodes();
+  
+  /**
+   * Number of stale data nodes
+   * @return number of stale data nodes
+   */
+  public int getNumStaleDataNodes();
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java

@@ -82,9 +82,9 @@ public class DatanodeRegistration extends DatanodeID
   public String toString() {
     return getClass().getSimpleName()
       + "(" + getIpAddr()
-      + ", storageID=" + storageID
-      + ", infoPort=" + infoPort
-      + ", ipcPort=" + ipcPort
+      + ", storageID=" + getStorageID()
+      + ", infoPort=" + getInfoPort()
+      + ", ipcPort=" + getIpcPort()
       + ", storageInfo=" + storageInfo
       + ")";
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java

@@ -165,7 +165,7 @@ class ImageLoaderCurrent implements ImageLoader {
 
       if (LayoutVersion.supports(Feature.FSIMAGE_COMPRESSION, imageVersion)) {
         boolean isCompressed = in.readBoolean();
-        v.visit(ImageElement.IS_COMPRESSED, imageVersion);
+        v.visit(ImageElement.IS_COMPRESSED, String.valueOf(isCompressed));
         if (isCompressed) {
           String codecClassName = Text.readString(in);
           v.visit(ImageElement.COMPRESS_CODEC, codecClassName);

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/conffile

@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-/etc/hadoop

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/control

@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Package: hadoop-hdfs
-Version: @version@
-Section: misc
-Priority: optional
-Provides: hadoop-hdfs
-Architecture: all
-Depends: openjdk-6-jre-headless, hadoop-common
-Maintainer: Apache Software Foundation <general@hadoop.apache.org>
-Description: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing.
-Distribution: development

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/postinst

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hdfs-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/postrm

@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-/usr/sbin/groupdel hadoop 2> /dev/null >dev/null
-exit 0

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/preinst

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-/usr/sbin/useradd --comment "Hadoop HDFS" -u 201 --shell /bin/bash -M -r --groups hadoop --home /var/lib/hadoop/hdfs hdfs 2> /dev/null || :

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/hadoop.control/prerm

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bash /usr/sbin/update-hdfs-env.sh \
-  --prefix=/usr \
-  --bin-dir=/usr/bin \
-  --sbin-dir=/usr/sbin \
-  --conf-dir=/etc/hadoop \
-  --log-dir=/var/log/hadoop \
-  --pid-dir=/var/run/hadoop \
-  --uninstal

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/init.d/hadoop-datanode

@@ -1,142 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-datanode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-datanode: start and stop the Apache Hadoop Data Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-datanode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-datanode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Data Node server not in use (/etc/hadoop/hadoop-datanode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-datanode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0

hadoop-hdfs-project/hadoop-hdfs/src/main/packages/deb/init.d/hadoop-namenode

@@ -1,154 +0,0 @@
-#! /bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BEGIN INIT INFO
-# Provides:		hadoop-namenode	
-# Required-Start:	$remote_fs $syslog
-# Required-Stop:	$remote_fs $syslog
-# Default-Start:	2 3 4 5
-# Default-Stop:		
-# Short-Description:	Apache Hadoop Name Node server
-### END INIT INFO
-
-set -e
-
-# /etc/init.d/hadoop-namenode: start and stop the Apache Hadoop Name Node daemon
-
-test -x /usr/bin/hadoop || exit 0
-( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0
-
-umask 022
-
-if test -f /etc/default/hadoop-env.sh; then
-    . /etc/default/hadoop-env.sh
-fi
-
-. /lib/lsb/init-functions
-
-# Are we running from init?
-run_by_init() {
-    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
-}
-
-check_for_no_start() {
-    # forget it if we're trying to start, and /etc/hadoop/hadoop-namenode_not_to_be_run exists
-    if [ -e /etc/hadoop/hadoop-namenode_not_to_be_run ]; then 
-	if [ "$1" = log_end_msg ]; then
-	    log_end_msg 0
-	fi
-	if ! run_by_init; then
-	    log_action_msg "Apache Hadoop Name Node server not in use (/etc/hadoop/hadoop-namenode_not_to_be_run)"
-	fi
-	exit 0
-    fi
-}
-
-check_privsep_dir() {
-    # Create the PrivSep empty dir if necessary
-    if [ ! -d ${HADOOP_PID_DIR} ]; then
-	mkdir -p ${HADOOP_PID_DIR}
-        chown root:hadoop ${HADOOP_PID_DIR}
-	chmod 0775 ${HADOOP_PID_DIR} 
-    fi
-}
-
-format() {
-    su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format' hdfs
-}
-
-export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
-
-case "$1" in
-  start)
-	check_privsep_dir
-	check_for_no_start
-	log_daemon_msg "Starting Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  stop)
-	log_daemon_msg "Stopping Apache Hadoop Name Node server" "hadoop-namenode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  format)
-	log_daemon_msg "Formatting Apache Hadoop Name Node" "hadoop-namenode"
-	format
-	if [ $? -eq 0 ]; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-  restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-	    log_end_msg 0
-	else
-	    log_end_msg 1
-	fi
-	;;
-
-  try-restart)
-	check_privsep_dir
-	log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode"
-	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid
-	RET="$?"
-	set -e
-	case $RET in
-	    0)
-		# old daemon stopped
-		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then
-		    log_end_msg 0
-		else
-		    log_end_msg 1
-		fi
-		;;
-	    1)
-		# daemon not running
-		log_progress_msg "(not running)"
-		log_end_msg 0
-		;;
-	    *)
-		# failed to stop
-		log_progress_msg "(failed to stop)"
-		log_end_msg 1
-		;;
-	esac
-	;;
-
-  status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid ${JAVA_HOME}/bin/java hadoop-namenode && exit 0 || exit $?
-	;;
-
-  *)
-	log_action_msg "Usage: /etc/init.d/hadoop-namenode {start|stop|restart|try-restart|status}"
-	exit 1
-esac
-
-exit 0