Complete commit of prior merge.

The previous merge accidentally only committed the hdfs project instead of common and MR
as well.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-347@1463203 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -87,6 +87,33 @@ Maven build goals:
   * -Dtest.exclude=<TESTCLASSNAME>
   * -Dtest.exclude.pattern=**/<TESTCLASSNAME1>.java,**/<TESTCLASSNAME2>.java
 
+----------------------------------------------------------------------------------
+Building components separately
+
+If you are building a submodule directory, all the hadoop dependencies this
+submodule has will be resolved as all other 3rd party dependencies. This is,
+from the Maven cache or from a Maven repository (if not available in the cache
+or the SNAPSHOT 'timed out').
+An alternative is to run 'mvn install -DskipTests' from Hadoop source top
+level once; and then work from the submodule. Keep in mind that SNAPSHOTs
+time out after a while, using the Maven '-nsu' will stop Maven from trying
+to update SNAPSHOTs from external repos.
+
+----------------------------------------------------------------------------------
+Importing projects to eclipse
+
+When you import the project to eclipse, install hadoop-maven-plugins at first.
+
+  $ cd hadoop-maven-plugins
+  $ mvn install
+
+Then, generate eclipse project files.
+
+  $ mvn eclipse:eclipse -DskipTests
+
+At last, import to eclipse by specifying the root directory of the project via
+[File] > [Import] > [Existing Projects into Workspace].
+
 ----------------------------------------------------------------------------------
 Building distributions:
 
@@ -111,3 +138,69 @@ Create a local staging version of the website (in /tmp/hadoop-site)
   $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
 
 ----------------------------------------------------------------------------------
+
+Building on OS/X
+
+----------------------------------------------------------------------------------
+
+Hadoop does not build on OS/X with Java 7.
+see: https://issues.apache.org/jira/browse/HADOOP-9350
+
+----------------------------------------------------------------------------------
+
+Building on Windows
+
+----------------------------------------------------------------------------------
+Requirements:
+
+* Windows System
+* JDK 1.6
+* Maven 3.0
+* Windows SDK or Visual Studio 2010 Professional
+* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
+* Findbugs 1.3.9 (if running findbugs)
+* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
+* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+
+If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
+Do not use Visual Studio Express.  It does not support compiling for 64-bit,
+which is problematic if running a 64-bit system.  The Windows SDK is free to
+download here:
+
+http://www.microsoft.com/en-us/download/details.aspx?id=8279
+
+----------------------------------------------------------------------------------
+Building:
+
+Keep the source code tree in a short path to avoid running into problems related
+to Windows maximum path length limitation.  (For example, C:\hdc).
+
+Run builds from a Windows SDK Command Prompt.  (Start, All Programs,
+Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt.)
+
+JAVA_HOME must be set, and the path must not contain spaces.  If the full path
+would contain spaces, then use the Windows short path instead.
+
+You must set the Platform environment variable to either x64 or Win32 depending
+on whether you're running a 64-bit or 32-bit system.  Note that this is
+case-sensitive.  It must be "Platform", not "PLATFORM" or "platform".
+Environment variables on Windows are usually case-insensitive, but Maven treats
+them as case-sensitive.  Failure to set this environment variable correctly will
+cause msbuild to fail while building the native code in hadoop-common.
+
+set Platform=x64 (when building on a 64-bit system)
+set Platform=Win32 (when building on a 32-bit system)
+
+Several tests require that the user must have the Create Symbolic Links
+privilege.
+
+All Maven goals are the same as described above with the exception that
+native code is built by enabling the 'native-win' Maven profile. -Pnative-win 
+is enabled by default when building on Windows since the native components 
+are required (not optional) on Windows.
+
+----------------------------------------------------------------------------------
+Building distributions:
+
+ * Build distribution with native code    : mvn package [-Pdist][-Pdocs][-Psrc][-Dtar]
+

dev-support/findHangingTest.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# script to find hanging test from Jenkins build output
+# usage: ./findHangingTest.sh <url of Jenkins build console>
+#
+`curl -k -o jenkins.out "$1"`
+expecting=Running
+cat jenkins.out | while read line; do
+ if [[ "$line" =~ "Running org.apache.hadoop" ]]; then
+  if [[ "$expecting" =~ "Running" ]]; then 
+   expecting=Tests
+  else
+   echo "Hanging test: $prevLine"
+  fi
+ fi
+ if [[ "$line" =~ "Tests run" ]]; then
+  expecting=Running
+ fi
+ if [[ "$line" =~ "Forking command line" ]]; then
+  a=$line
+ else
+  prevLine=$line
+ fi
+done

dev-support/test-patch.sh

@@ -418,8 +418,8 @@ checkJavadocWarnings () {
   echo ""
   echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
 
-  #There are 6 warnings that are caused by things that are caused by using sun internal APIs.
-  OK_JAVADOC_WARNINGS=6;
+  #There are 11 warnings that are caused by things that are caused by using sun internal APIs.
+  OK_JAVADOC_WARNINGS=11;
   ### if current warnings greater than OK_JAVADOC_WARNINGS
   if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -26,6 +26,9 @@
       <outputDirectory>/bin</outputDirectory>
       <excludes>
         <exclude>*.sh</exclude>
+        <exclude>*-config.cmd</exclude>
+        <exclude>start-*.cmd</exclude>
+        <exclude>stop-*.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -38,6 +41,7 @@
       <outputDirectory>/libexec</outputDirectory>
       <includes>
         <include>*-config.sh</include>
+        <include>*-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -46,9 +50,13 @@
       <outputDirectory>/sbin</outputDirectory>
       <includes>
         <include>*.sh</include>
+        <include>*.cmd</include>
       </includes>
       <excludes>
         <exclude>hadoop-config.sh</exclude>
+        <exclude>hadoop.cmd</exclude>
+        <exclude>hdfs.cmd</exclude>
+        <exclude>hadoop-config.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -33,6 +33,7 @@
       <outputDirectory>bin</outputDirectory>
       <includes>
         <include>yarn</include>
+        <include>yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -41,6 +42,7 @@
       <outputDirectory>libexec</outputDirectory>
       <includes>
         <include>yarn-config.sh</include>
+        <include>yarn-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -52,6 +54,8 @@
         <include>yarn-daemons.sh</include>
         <include>start-yarn.sh</include>
         <include>stop-yarn.sh</include>
+        <include>start-yarn.cmd</include>
+        <include>stop-yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -121,7 +125,7 @@
       </includes>
       <binaries>
         <attachmentClassifier>tests</attachmentClassifier>
-        <outputDirectory>share/hadoop/${hadoop.component}</outputDirectory>
+        <outputDirectory>share/hadoop/${hadoop.component}/test</outputDirectory>
         <includeDependencies>false</includeDependencies>
         <unpack>false</unpack>
       </binaries>

hadoop-client/pom.xml

@@ -115,6 +115,14 @@
           <groupId>net.java.dev.jets3t</groupId>
           <artifactId>jets3t</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.jcraft</groupId>
+          <artifactId>jsch</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-el</groupId>
+          <artifactId>commons-el</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -131,6 +139,34 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.mortbay.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-server</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet.jsp</groupId>
+          <artifactId>jsp-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -171,6 +207,10 @@
           <groupId>jline</groupId>
           <artifactId>jline</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -207,6 +247,18 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -247,6 +299,14 @@
           <groupId>com.google.inject.extensions</groupId>
           <artifactId>guice-servlet</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -271,8 +331,25 @@
           <groupId>com.google.inject.extensions</groupId>
           <artifactId>guice-servlet</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-annotations</artifactId>
+      <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
+    
   </dependencies>
 
 </project>

hadoop-common-project/hadoop-annotations/pom.xml

@@ -38,4 +38,24 @@
     </dependency>
   </dependencies>
 
+  <profiles>
+    <profile>
+      <id>os.linux</id>
+      <activation>
+        <os>
+          <family>!Mac</family>
+        </os>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+          <version>1.6</version>
+          <scope>system</scope>
+          <systemPath>${java.home}/../lib/tools.jar</systemPath>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
 </project>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java

@@ -383,9 +383,25 @@ public class KerberosName {
    * @param ruleString the rules string.
    */
   public static void setRules(String ruleString) {
-    rules = parseRules(ruleString);
+    rules = (ruleString != null) ? parseRules(ruleString) : null;
   }
 
+  /**
+   * Get the rules.
+   * @return String of configured rules, or null if not yet configured
+   */
+  public static String getRules() {
+    String ruleString = null;
+    if (rules != null) {
+      StringBuilder sb = new StringBuilder();
+      for (Rule rule : rules) {
+        sb.append(rule.toString()).append("\n");
+      }
+      ruleString = sb.toString().trim();
+    }
+    return ruleString;
+  }
+  
   /**
    * Indicates if the name rules have been set.
    * 

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -8,6 +8,16 @@ Trunk (Unreleased)
     FSDataOutputStream.sync() and Syncable.sync().  (szetszwo)
 
     HADOOP-8886. Remove KFS support. (eli)
+    
+    HADOOP-9163 The rpc msg in ProtobufRpcEngine.proto should be moved out to
+    avoid an extra copy (Sanjay Radia)
+
+    HADOOP-9151 Include RPC error info in RpcResponseHeader instead of sending
+    it separately (sanjay Radia)
+
+    HADOOP-9380 Add totalLength to rpc response  (sanjay Radia)
+
+    HADOOP-9194. RPC Support for QoS. (Junping Du via llu)
 
   NEW FEATURES
     
@@ -20,6 +30,10 @@ Trunk (Unreleased)
     HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation
     of NetworkTopology.  (Junping Du via szetszwo)
 
+    HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows
+    Azure environments. (See breakdown of tasks below for subtasks and
+    contributors)
+
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
@@ -78,9 +92,6 @@ Trunk (Unreleased)
     HADOOP-8367 Improve documentation of declaringClassProtocolName in 
     rpc headers. (Sanjay Radia)
 
-    HADOOP-8415. Add getDouble() and setDouble() in
-    org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
-
     HADOOP-7659. fs -getmerge isn't guaranteed to work well over non-HDFS
     filesystems (harsh)
 
@@ -150,6 +161,8 @@ Trunk (Unreleased)
 
     HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia)
 
+	HADOOP-9258 Add stricter tests to FileSystemContractTestBase (stevel)
+
   BUG FIXES
 
     HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
@@ -326,6 +339,31 @@ Trunk (Unreleased)
     HADOOP-9264. Port change to use Java untar API on Windows from 
     branch-1-win to trunk. (Chris Nauroth via suresh)
 
+    HADOOP-9393. TestRPC fails with JDK7. (Andrew Wang via atm)
+
+    HADOOP-9394. Port findHangingTest.sh from HBase to Hadoop. (Andrew Wang
+    via atm)
+
+    HADOOP-9099. NetUtils.normalizeHostName fails on domains where 
+    UnknownHost resolves to an IP address. (Ivan Mitic via suresh)
+
+    HADOOP-9397. Incremental dist tar build fails (Chris Nauroth via jlowe)
+
+    HADOOP-9405. TestGridmixSummary#testExecutionSummarizer is broken. (Andrew
+    Wang via atm)
+
+    HADOOP-9431 TestSecurityUtil#testLocalHostNameForNullOrWild on systems where hostname
+    contains capital letters  (Chris Nauroth via sanjay)
+
+    HADOOP-9261 S3n filesystem can move a directory under itself -and so lose data
+    (fixed in HADOOP-9258) (stevel)
+
+    HADOOP-9265 S3 blockstore filesystem breaks part of the Filesystem contract
+    (fixed in HADOOP-9258) (stevel)
+
+    HADOOP-9433 TestLocalFileSystem#testHasFileDescriptor leaks file handle
+    (Chris Nauroth via sanjay)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -334,7 +372,138 @@ Trunk (Unreleased)
 
     HADOOP-9190. packaging docs is broken. (Andy Isaacson via atm)
 
-Release 2.0.4-beta - UNRELEASED
+  BREAKDOWN OF HADOOP-8562 SUBTASKS
+
+    HADOOP-8924. Hadoop Common creating package-info.java must not depend on
+    sh. (Chris Nauroth via suresh)
+
+    HADOOP-8945. Merge winutils from branch-1-win to branch-trunk-win.
+    (Bikas Saha, Chuan Liu, Giridharan Kesavan, Ivan Mitic, and Steve Maine
+    ported by Chris Nauroth via suresh)
+
+    HADOOP-8946. winutils: compile codebase during Maven build on 
+    branch-trunk-win. (Chris Nauroth via suresh)
+
+    HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to 
+    branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, 
+    Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, 
+    Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh)
+
+    HADOOP-8954. "stat" executable not found on Windows. (Bikas Saha, Ivan Mitic
+    ported by Chris Narouth via suresh)
+
+    HADOOP-8959. TestUserGroupInformation fails on Windows due to "id" executable
+    not found. (Bikas Saha, Ivan Mitic, ported by Chris Narouth via suresh)
+
+    HADOOP-8955. "chmod" executable not found on Windows.
+    (Chris Nauroth via suresh)
+
+    HADOOP-8960. TestMetricsServlet fails on Windows. (Ivan Mitic via suresh)
+
+    HADOOP-8961. GenericOptionsParser URI parsing failure on Windows.
+    (Ivan Mitic via suresh)
+
+    HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via 
+    suresh)
+
+    HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple
+    test suites to fail. (Chris Nauroth via suresh)
+
+    HADOOP-8978. TestTrash fails on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-8979. TestHttpServer fails on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-8953. Shell PathData parsing failures on Windows. (Arpit Agarwal via
+    suresh)
+
+    HADOOP-8975. TestFileContextResolveAfs fails on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-8977. Multiple FsShell test failures on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, 
+    Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh)
+
+    HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-9011. saveVersion.py does not include branch in version annotation.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9110. winutils ls off-by-one error indexing MONTHS array can cause 
+    access violation. (Chris Nauroth via suresh)
+
+    HADOOP-9056. Build native library on Windows. (Chuan Liu, Arpit Agarwal via
+    suresh)
+
+    HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh)
+
+    HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, 
+    and Bikas Saha via suresh)
+
+    HADOOP-9146. Fix sticky bit regression on branch-trunk-win.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9266. Fix javac, findbugs, and release audit warnings on
+    branch-trunk-win. (Chris Nauroth via suresh)
+
+    HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via
+    szetszwo)
+
+    HADOOP-9271. Revert Python build scripts from branch-trunk-win.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9313. Remove spurious mkdir from hadoop-config.cmd.
+    (Ivan Mitic via suresh)
+
+    HADOOP-9309. Test failures on Windows due to UnsatisfiedLinkError
+    in NativeCodeLoader#buildSupportsSnappy. (Arpit Agarwal via suresh)
+
+    HADOOP-9347. Add instructions to BUILDING.txt describing how to
+    build on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-9348. Address TODO in winutils to add more command line usage
+    and examples. (Chris Nauroth via suresh)
+
+    HADOOP-9354. Windows native project files missing license headers.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9356. Remove remaining references to cygwin/cygpath from scripts.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9232. JniBasedUnixGroupsMappingWithFallback fails on Windows
+    with UnsatisfiedLinkError. (Ivan Mitic via suresh)
+
+    HADOOP-9368. Add timeouts to new tests in branch-trunk-win.
+    (Arpit Agarwal via suresh)
+
+    HADOOP-9373. Merge CHANGES.branch-trunk-win.txt to CHANGES.txt.
+    (suresh)
+
+    HADOOP-9372. Fix bad timeout annotations on tests.
+    (Arpit Agarwal via suresh)
+
+    HADOOP-9376. TestProxyUserFromEnv fails on a Windows domain joined machine.
+    (Ivan Mitic via suresh)
+
+    HADOOP-9365. TestHAZKUtil fails on Windows. (Ivan Mitic via suresh)
+
+    HADOOP-9364. PathData#expandAsGlob does not return correct results for
+    absolute paths on Windows. (Ivan Mitic via suresh)
+
+    HADOOP-8973. DiskChecker cannot reliably detect an inaccessible disk on
+    Windows with NTFS ACLs. (Chris Nauroth via suresh)
+
+    HADOOP-9388. TestFsShellCopy fails on Windows. (Ivan Mitic via suresh)
+
+    HADOOP-9387. Fix DF so that it won't execute a shell command on Windows
+    to compute the file system/mount point.  (Ivan Mitic via szetszwo)
+
+    HADOOP-9353. Activate native-win maven profile by default on Windows.
+    (Arpit Agarwal via szetszwo)
+    
+Release 2.0.5-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
@@ -342,6 +511,9 @@ Release 2.0.4-beta - UNRELEASED
 
     HADOOP-9283. Add support for running the Hadoop client on AIX. (atm)
 
+    HADOOP-8415. Add getDouble() and setDouble() in
+    org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
+
   IMPROVEMENTS
 
     HADOOP-9253. Capture ulimit info in the logs at service start time.
@@ -352,8 +524,23 @@ Release 2.0.4-beta - UNRELEASED
 
     HADOOP-9117. replace protoc ant plugin exec with a maven plugin. (tucu)
 
+    HADOOP-9279. Document the need to build hadoop-maven-plugins for
+    eclipse and separate project builds. (Tsuyoshi Ozawa via suresh)
+
+    HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh)
+
+    HADOOP-9343. Allow additional exceptions through the RPC layer. (sseth)
+
+    HADOOP-9318. When exiting on a signal, print the signal name first. (Colin
+    Patrick McCabe via atm)
+
+    HADOOP-9358. "Auth failed" log should include exception string (todd)
+
   OPTIMIZATIONS
 
+    HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
+    (todd)
+
   BUG FIXES
 
     HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh)
@@ -373,6 +560,76 @@ Release 2.0.4-beta - UNRELEASED
 
     HADOOP-9304. remove addition of avro genreated-sources dirs to build. (tucu)
 
+    HADOOP-9267. hadoop -help, -h, --help should show usage instructions.
+    (Andrew Wang via atm)
+
+    HADOOP-8569. CMakeLists.txt: define _GNU_SOURCE and _LARGEFILE_SOURCE.
+    (Colin Patrick McCabe via atm)
+
+    HADOOP-9323. Fix typos in API documentation. (suresh)
+
+    HADOOP-7487. DF should throw a more reasonable exception when mount cannot
+    be determined. (Andrew Wang via atm)
+
+    HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern.
+    (Arpit Gupta via suresh)
+
+    HADOOP-9342. Remove jline from distribution. (thw via tucu)
+
+    HADOOP-9230. TestUniformSizeInputFormat fails intermittently. 
+    (kkambatl via tucu)
+
+    HADOOP-9349. Confusing output when running hadoop version from one hadoop 
+    installation when HADOOP_HOME points to another. (sandyr via tucu)
+
+    HADOOP-9337. org.apache.hadoop.fs.DF.getMount() does not work on Mac OS.
+    (Ivan A. Veselovsky via atm)
+
+    HADOOP-9369. DNS#reverseDns() can return hostname with . appended at the
+    end. (Karthik Kambatla via atm)
+
+    HADOOP-9379. capture the ulimit info after printing the log to the 
+    console. (Arpit Gupta via suresh)
+
+    HADOOP-9399. protoc maven plugin doesn't work on mvn 3.0.2 (todd)
+
+    HADOOP-9407. commons-daemon 1.0.3 dependency has bad group id causing
+    build issues. (Sangjin Lee via suresh)
+
+    HADOOP-9299.  kerberos name resolution is kicking in even when kerberos
+    is not configured (daryn)
+
+    HADOOP-9430. TestSSLFactory fails on IBM JVM. (Amir Sanjar via suresh)
+
+    HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
+    idle time. (Kai Zheng via atm)
+
+    HADOOP-9357. Fallback to default authority if not specified in FileContext.
+    (Andrew Wang via eli)
+
+Release 2.0.4-alpha - UNRELEASED
+
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+  IMPROVEMENTS
+
+  OPTIMIZATIONS
+
+  BUG FIXES
+
+    HADOOP-9406. hadoop-client leaks dependency on JDK tools jar. (tucu)
+
+    HADOOP-9301. hadoop client servlet/jsp/jetty/tomcat JARs creating
+    conflicts in Oozie & HttpFS. (tucu)
+
+    HADOOP-9408. misleading description for net.topology.table.file.name
+    property in core-default.xml. (rajeshbabu via suresh)
+
+    HADOOP-9444. Modify hadoop-policy.xml to replace unexpanded variables to a
+    default value of '*'. (Roman Shaposhnik via vinodkv)
+
 Release 2.0.3-alpha - 2013-02-06 
 
   INCOMPATIBLE CHANGES
@@ -633,15 +890,15 @@ Release 2.0.3-alpha - 2013-02-06
     HADOOP-9124. SortedMapWritable violates contract of Map interface for
     equals() and hashCode(). (Surenkumar Nihalani via tomwhite)
 
+    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
+    HarFileSystem. (Chris Nauroth via szetszwo)
+
     HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and
     the synchronization of limitDecimalTo2(double) can be avoided.  (szetszwo)
 
     HADOOP-9260. Hadoop version may be not correct when starting name node or
     data node. (Chris Nauroth via jlowe)
 
-    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
-    HarFileSystem. (Chris Nauroth via szetszwo)
-
     HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via
     suresh)
 
@@ -1348,10 +1605,23 @@ Release 0.23.7 - UNRELEASED
     permissions (Ivan A. Veselovsky via bobby)
 
     HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure
-    (Ivan A. Veselovsky via bobby) 
+    (Ivan A. Veselovsky via bobby)
+
+    HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp
+    via kihwal)
+
+    HADOOP-9352. Expose UGI.setLoginUser for tests (daryn)
+
+    HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via
+    jeagles)
+
+    HADOOP-9374. Add tokens from -tokenCacheFile into UGI (daryn)
 
   OPTIMIZATIONS
 
+    HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via
+    jlowe)
+
   BUG FIXES
 
     HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via
@@ -1360,6 +1630,9 @@ Release 0.23.7 - UNRELEASED
     HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage
     option (Andy Isaacson via tgraves)
 
+    HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp via 
+    kihwal)
+
 Release 0.23.6 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -2598,6 +2871,9 @@ Release 0.23.0 - 2011-11-01
     HADOOP-7797. Fix top-level pom.xml to refer to correct staging maven
     repository. (omalley via acmurthy) 
 
+    HADOOP-7101. UserGroupInformation.getCurrentUser() fails when called from
+    non-Hadoop JAAS context. (todd)
+
 Release 0.22.1 - Unreleased
 
   INCOMPATIBLE CHANGES
@@ -3055,9 +3331,6 @@ Release 0.22.0 - 2011-11-29
 
     HADOOP-7093. Servlets should default to text/plain (todd)
 
-    HADOOP-7101. UserGroupInformation.getCurrentUser() fails when called from
-    non-Hadoop JAAS context. (todd)
-
     HADOOP-7089. Fix link resolution logic in hadoop-config.sh. (eli)
 
     HADOOP-7046. Fix Findbugs warning in Configuration. (Po Cheung via shv)

hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml

@@ -308,4 +308,11 @@
        <Method name="removeRenewAction" />
        <Bug pattern="BC_UNCONFIRMED_CAST" />
      </Match>
+     
+     <!-- Inconsistent synchronization flagged by findbugs is not valid. -->
+     <Match>
+       <Class name="org.apache.hadoop.ipc.Client$Connection" />
+       <Field name="in" />
+       <Bug pattern="IS2_INCONSISTENT_SYNC" />
+     </Match>
  </FindBugsFilter>

hadoop-common-project/hadoop-common/pom.xml

@@ -219,6 +219,14 @@
       <artifactId>zookeeper</artifactId>
       <version>3.4.2</version>
       <exclusions>
+        <exclusion>
+          <groupId>jline</groupId>
+          <artifactId>jline</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jboss.netty</groupId>
+          <artifactId>netty</artifactId>
+        </exclusion>
         <exclusion>
           <!-- otherwise seems to drag in junit 3.8.1 via jline -->
           <groupId>junit</groupId>
@@ -460,6 +468,7 @@
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
+        <require.bzip2>false</require.bzip2>
         <snappy.prefix></snappy.prefix>
         <snappy.lib></snappy.lib>
         <snappy.include></snappy.include>
@@ -467,6 +476,28 @@
       </properties>
       <build>
         <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>mac</family>
+                      <family>unix</family>
+                      <message>native build only supported on Mac or Unix</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>native-maven-plugin</artifactId>
@@ -481,6 +512,8 @@
                   <javahClassNames>
                     <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Compressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
                     <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
                     <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
@@ -507,7 +540,7 @@
                 <configuration>
                   <target>
                     <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                       <arg line="VERBOSE=1"/>
@@ -538,6 +571,106 @@
       </build>
     </profile>
 
+    <profile>
+      <id>native-win</id>
+      <activation>
+        <os>
+          <family>Windows</family>
+        </os>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>windows</family>
+                      <message>native-win build only supported on Windows</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>native-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>javah</goal>
+                </goals>
+                <configuration>
+                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+                  <javahClassNames>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+                  </javahClassNames>
+                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>compile-ms-winutils</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/winutils/winutils.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+              <execution>
+                <id>compile-ms-native-dll</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/native/native.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!-- profile that starts ApacheDS KDC server -->
     <profile>
       <id>startKdc</id>

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -90,12 +90,30 @@ find_package(ZLIB REQUIRED)
 SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
 set(D main/native/src/org/apache/hadoop)
 set(T main/native/src/test/org/apache/hadoop)
 
 GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
 
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1")
+find_package(BZip2 QUIET)
+if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+    GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
+    set(BZIP2_SOURCE_FILES
+          "${D}/io/compress/bzip2/Bzip2Compressor.c"
+          "${D}/io/compress/bzip2/Bzip2Decompressor.c")
+else (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+    set(BZIP2_SOURCE_FILES "")
+    set(BZIP2_INCLUDE_DIR "")
+    IF(REQUIRE_BZIP2)
+        MESSAGE(FATAL_ERROR "Required bzip2 library and/or header files could not be found.")
+    ENDIF(REQUIRE_BZIP2)
+endif (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+
 INCLUDE(CheckFunctionExists)
 INCLUDE(CheckCSourceCompiles)
 INCLUDE(CheckLibraryExists)
@@ -135,6 +153,7 @@ include_directories(
     ${CMAKE_BINARY_DIR}
     ${JNI_INCLUDE_DIRS}
     ${ZLIB_INCLUDE_DIRS}
+    ${BZIP2_INCLUDE_DIR}
     ${SNAPPY_INCLUDE_DIR}
     ${D}/util
 )
@@ -154,6 +173,7 @@ add_dual_library(hadoop
     ${SNAPPY_SOURCE_FILES}
     ${D}/io/compress/zlib/ZlibCompressor.c
     ${D}/io/compress/zlib/ZlibDecompressor.c
+    ${BZIP2_SOURCE_FILES}
     ${D}/io/nativeio/NativeIO.c
     ${D}/io/nativeio/errno_enum.c
     ${D}/io/nativeio/file_descriptor.c

hadoop-common-project/hadoop-common/src/config.h.cmake

@@ -19,6 +19,7 @@
 #define CONFIG_H
 
 #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
+#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
 #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
 #cmakedefine HAVE_SYNC_FILE_RANGE
 #cmakedefine HAVE_POSIX_FADVISE

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -50,6 +50,12 @@ fi
 
 COMMAND=$1
 case $COMMAND in
+  # usage flags
+  --help|-help|-h)
+    print_usage
+    exit
+    ;;
+
   #hdfs commands
   namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups)
     echo "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1>&2
@@ -85,9 +91,6 @@ case $COMMAND in
     ;;
 
   classpath)
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     echo $CLASSPATH
     exit
     ;;
@@ -126,9 +129,6 @@ case $COMMAND in
     #make sure security appender is turned off
     HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
 
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     export CLASSPATH=$CLASSPATH
     exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
     ;;

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd

@@ -0,0 +1,292 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem included in all the hadoop scripts with source command
+@rem should not be executable directly
+@rem also should not be passed any arguments, since we need original %*
+
+if not defined HADOOP_COMMON_DIR (
+  set HADOOP_COMMON_DIR=share\hadoop\common
+)
+if not defined HADOOP_COMMON_LIB_JARS_DIR (
+  set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib
+)
+if not defined HADOOP_COMMON_LIB_NATIVE_DIR (
+  set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native
+)
+if not defined HDFS_DIR (
+  set HDFS_DIR=share\hadoop\hdfs
+)
+if not defined HDFS_LIB_JARS_DIR (
+  set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib
+)
+if not defined YARN_DIR (
+  set YARN_DIR=share\hadoop\yarn
+)
+if not defined YARN_LIB_JARS_DIR (
+  set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib
+)
+if not defined MAPRED_DIR (
+  set MAPRED_DIR=share\hadoop\mapreduce
+)
+if not defined MAPRED_LIB_JARS_DIR (
+  set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib
+)
+
+@rem the root of the Hadoop installation
+set HADOOP_HOME=%~dp0
+for %%i in (%HADOOP_HOME%.) do (
+  set HADOOP_HOME=%%~dpi
+)
+if "%HADOOP_HOME:~-1%" == "\" (
+  set HADOOP_HOME=%HADOOP_HOME:~0,-1%
+)
+
+if not exist %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar (
+    @echo +================================================================+
+    @echo ^|      Error: HADOOP_HOME is not set correctly                   ^|
+    @echo +----------------------------------------------------------------+
+    @echo ^| Please set your HADOOP_HOME variable to the absolute path of   ^|
+    @echo ^| the directory that contains the hadoop distribution            ^|
+    @echo +================================================================+
+    exit /b 1
+)
+
+set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
+
+@rem
+@rem Allow alternate conf dir location.
+@rem
+
+if "%1" == "--config" (
+  set HADOOP_CONF_DIR=%2
+  shift
+  shift
+)
+
+@rem
+@rem check to see it is specified whether to use the slaves or the
+@rem masters file
+@rem
+
+if "%1" == "--hosts" (
+  set HADOOP_SLAVES=%HADOOP_CONF_DIR%\%2
+  shift
+  shift
+)
+
+if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
+  call %HADOOP_CONF_DIR%\hadoop-env.cmd
+)
+
+@rem
+@rem setup java environment variables
+@rem
+
+if not defined JAVA_HOME (
+  echo Error: JAVA_HOME is not set.
+  goto :eof
+)
+
+if not exist %JAVA_HOME%\bin\java.exe (
+  echo Error: JAVA_HOME is incorrectly set.
+  echo        Please update %HADOOP_HOME%\conf\hadoop-env.cmd
+  goto :eof
+)
+
+set JAVA=%JAVA_HOME%\bin\java
+@rem some Java parameters
+set JAVA_HEAP_MAX=-Xmx1000m
+
+@rem
+@rem check envvars which might override default args
+@rem
+
+if defined HADOOP_HEAPSIZE (
+  set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m
+)
+
+@rem
+@rem CLASSPATH initially contains %HADOOP_CONF_DIR%
+@rem
+
+set CLASSPATH=%HADOOP_CONF_DIR%
+
+if not defined HADOOP_COMMON_HOME (
+  if exist %HADOOP_HOME%\share\hadoop\common (
+    set HADOOP_COMMON_HOME=%HADOOP_HOME%
+  )
+)
+
+@rem
+@rem for releases, add core hadoop jar & webapps to CLASSPATH
+@rem
+
+if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%
+)
+
+if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\*
+
+@rem
+@rem add user-specified CLASSPATH last
+@rem
+
+if defined HADOOP_CLASSPATH (
+  if defined HADOOP_USER_CLASSPATH_FIRST (
+    set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
+  ) else (
+    set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
+  )
+)
+
+@rem
+@rem default log directory % file
+@rem
+
+if not defined HADOOP_LOG_DIR (
+  set HADOOP_LOG_DIR=%HADOOP_HOME%\logs
+)
+
+if not defined HADOOP_LOGFILE (
+  set HADOOP_LOGFILE=hadoop.log
+)
+
+if not defined HADOOP_ROOT_LOGGER (
+  set HADOOP_ROOT_LOGGER=INFO,console
+)
+
+@rem
+@rem default policy file for service-level authorization
+@rem
+
+if not defined HADOOP_POLICYFILE (
+  set HADOOP_POLICYFILE=hadoop-policy.xml
+)
+
+@rem
+@rem Determine the JAVA_PLATFORM
+@rem
+
+for /f "delims=" %%A in ('%JAVA% -Xmx32m %HADOOP_JAVA_PLATFORM_OPTS% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName') do set JAVA_PLATFORM=%%A
+@rem replace space with underscore
+set JAVA_PLATFORM=%JAVA_PLATFORM: =_%
+
+@rem
+@rem setup 'java.library.path' for native hadoop code if necessary
+@rem
+
+@rem Check if we're running hadoop directly from the build
+set JAVA_LIBRARY_PATH=
+if exist %HADOOP_CORE_HOME%\target\bin (
+  set JAVA_LIBRARY_PATH=%HADOOP_CORE_HOME%\target\bin
+)
+
+@rem For the distro case, check the bin folder
+if exist %HADOOP_CORE_HOME%\bin (
+  set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\bin
+)
+
+@rem
+@rem setup a default TOOL_PATH
+@rem
+set TOOL_PATH=%HADOOP_HOME%\share\hadoop\tools\lib\*
+
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_HOME%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER%
+
+if defined JAVA_LIBRARY_PATH (
+  set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE%
+
+@rem
+@rem Disable ipv6 as it can cause issues
+@rem
+
+set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
+
+@rem
+@rem put hdfs in classpath if present
+@rem
+
+if not defined HADOOP_HDFS_HOME (
+  if exist %HADOOP_HOME%\%HDFS_DIR% (
+    set HADOOP_HDFS_HOME=%HADOOP_HOME%
+  )
+)
+
+if exist %HADOOP_HDFS_HOME%\%HDFS_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%
+)
+
+if exist %HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\*
+
+@rem
+@rem put yarn in classpath if present
+@rem
+
+if not defined HADOOP_YARN_HOME (
+  if exist %HADOOP_HOME%\%YARN_DIR% (
+    set HADOOP_YARN_HOME=%HADOOP_HOME%
+  )
+)
+
+if exist %HADOOP_YARN_HOME%\%YARN_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%
+)
+
+if exist %HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\*
+
+@rem
+@rem put mapred in classpath if present AND different from YARN
+@rem
+
+if not defined HADOOP_MAPRED_HOME (
+  if exist %HADOOP_HOME%\%MAPRED_DIR% (
+    set HADOOP_MAPRED_HOME=%HADOOP_HOME%
+  )
+)
+
+if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" (
+
+  if exist %HADOOP_MAPRED_HOME%\%MAPRED_DIR%\webapps (
+    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%
+  )
+
+  if exist %HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR% (
+    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\*
+  )
+
+  set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
+)
+
+:eof

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh

@@ -112,12 +112,6 @@ if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then
   exit 1
 fi
 
-cygwin=false
-case "`uname`" in
-CYGWIN*) cygwin=true;;
-esac
-
-
 # check if net.ipv6.bindv6only is set to 1
 bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
 if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] && [ "$HADOOP_ALLOW_IPV6" != "yes" ]
@@ -209,13 +203,6 @@ fi
 # restore ordinary behaviour
 unset IFS
 
-# cygwin path translation
-if $cygwin; then
-  HADOOP_PREFIX=`cygpath -w "$HADOOP_PREFIX"`
-  HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"`
-  JAVA_LIBRARY_PATH=`cygpath -w "$JAVA_LIBRARY_PATH"`
-fi
-
 # setup 'java.library.path' for native-hadoop code if necessary
 
 if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then
@@ -232,11 +219,6 @@ fi
 # setup a default TOOL_PATH
 TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
 
-# cygwin path translation
-if $cygwin; then
-  JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
-fi
-
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
@@ -303,15 +285,3 @@ if [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" != "$HADOOP_YARN_HOME/$YARN_DIR" ] ; then
 
   CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR'/*'
 fi
-
-# cygwin path translation
-if $cygwin; then
-  HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`
-fi
-
-# cygwin path translation
-if $cygwin; then
-  TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
-fi
-
-

hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh

@@ -156,6 +156,7 @@ case $startStop in
     esac
     echo $! > $pid
     sleep 1
+    head "$log"
     # capture the ulimit output
     if [ "true" = "$starting_secure_dn" ]; then
       echo "ulimit -a for secure datanode user $HADOOP_SECURE_DN_USER" >> $log
@@ -165,7 +166,6 @@ case $startStop in
       echo "ulimit -a for user $USER" >> $log
       ulimit -a >> $log 2>&1
     fi
-    head -30 "$log"
     sleep 3;
     if ! ps -p $! > /dev/null ; then
       exit 1

hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd

@@ -0,0 +1,235 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+
+@rem This script runs the hadoop core commands. 
+
+@rem Environment Variables
+@rem
+@rem   JAVA_HOME        The java implementation to use.  Overrides JAVA_HOME.
+@rem
+@rem   HADOOP_CLASSPATH Extra Java CLASSPATH entries.
+@rem
+@rem   HADOOP_USER_CLASSPATH_FIRST      When defined, the HADOOP_CLASSPATH is
+@rem                                    added in the beginning of the global
+@rem                                    classpath. Can be defined, for example,
+@rem                                    by doing
+@rem                                    export HADOOP_USER_CLASSPATH_FIRST=true
+@rem
+@rem   HADOOP_HEAPSIZE  The maximum amount of heap to use, in MB.
+@rem                    Default is 1000.
+@rem
+@rem   HADOOP_OPTS      Extra Java runtime options.
+@rem
+@rem   HADOOP_CLIENT_OPTS         when the respective command is run.
+@rem   HADOOP_{COMMAND}_OPTS etc  HADOOP_JT_OPTS applies to JobTracker
+@rem                              for e.g.  HADOOP_CLIENT_OPTS applies to
+@rem                              more than one command (fs, dfs, fsck,
+@rem                              dfsadmin etc)
+@rem
+@rem   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
+@rem
+@rem   HADOOP_ROOT_LOGGER The root appender. Default is INFO,console
+@rem
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+call :updatepath %HADOOP_BIN_PATH%
+
+:main
+  setlocal enabledelayedexpansion
+
+  set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+  if not defined HADOOP_LIBEXEC_DIR (
+    set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+  )
+
+  call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+  if "%1" == "--config" (
+    shift
+    shift
+  )
+
+  set hadoop-command=%1
+  if not defined hadoop-command (
+      goto print_usage
+  )
+
+  call :make_command_arguments %*
+
+  set hdfscommands=namenode secondarynamenode datanode dfs dfsadmin fsck balancer fetchdt oiv dfsgroups
+  for %%i in ( %hdfscommands% ) do (
+    if %hadoop-command% == %%i set hdfscommand=true
+  )
+  if defined hdfscommand (
+    @echo DEPRECATED: Use of this script to execute hdfs command is deprecated. 1>&2
+    @echo Instead use the hdfs command for it. 1>&2
+    if exist %HADOOP_HDFS_HOME%\bin\hdfs.cmd (
+      call %HADOOP_HDFS_HOME%\bin\hdfs.cmd %*
+      goto :eof
+    ) else if exist %HADOOP_HOME%\bin\hdfs.cmd (
+      call %HADOOP_HOME%\bin\hdfs.cmd %*
+      goto :eof
+    ) else (
+      echo HADOOP_HDFS_HOME not found!
+      goto :eof
+    )
+  )
+
+  set mapredcommands=pipes job queue mrgroups mradmin jobtracker tasktracker
+  for %%i in ( %mapredcommands% ) do (
+    if %hadoop-command% == %%i set mapredcommand=true  
+  )
+  if defined mapredcommand (
+    @echo DEPRECATED: Use of this script to execute mapred command is deprecated. 1>&2
+    @echo Instead use the mapred command for it. 1>&2
+    if exist %HADOOP_MAPRED_HOME%\bin\mapred.cmd (
+      call %HADOOP_MAPRED_HOME%\bin\mapred.cmd %*
+      goto :eof
+    ) else if exist %HADOOP_HOME%\bin\mapred.cmd (
+      call %HADOOP_HOME%\bin\mapred.cmd %*
+      goto :eof
+    ) else (
+      echo HADOOP_MAPRED_HOME not found!
+      goto :eof
+    )
+  )
+
+  if %hadoop-command% == classpath (
+    @echo %CLASSPATH%
+    goto :eof
+  )
+  
+  set corecommands=fs version jar distcp daemonlog archive
+  for %%i in ( %corecommands% ) do (
+    if %hadoop-command% == %%i set corecommand=true  
+  )
+  if defined corecommand (
+    call :%hadoop-command%
+  ) else (
+    set CLASSPATH=%CLASSPATH%;%CD%
+    set CLASS=%hadoop-command%
+  )
+
+  set path=%PATH%;%HADOOP_BIN_PATH%
+
+  @rem Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
+  set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
+
+  @rem make sure security appender is turned off
+  if not defined HADOOP_SECURITY_LOGGER (
+    set HADOOP_SECURITY_LOGGER=INFO,NullAppender
+  )
+  set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER%
+
+  call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hadoop-command-arguments%
+
+  goto :eof
+
+:fs 
+  set CLASS=org.apache.hadoop.fs.FsShell
+  goto :eof
+
+:version 
+  set CLASS=org.apache.hadoop.util.VersionInfo
+  goto :eof
+
+:jar
+  set CLASS=org.apache.hadoop.util.RunJar
+  goto :eof
+
+:distcp
+  set CLASS=org.apache.hadoop.tools.DistCp
+  set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
+  goto :eof
+
+:daemonlog
+  set CLASS=org.apache.hadoop.log.LogLevel
+  goto :eof
+
+:archive
+  set CLASS=org.apache.hadoop.tools.HadoopArchives
+  set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
+  goto :eof
+
+:updatepath
+  set path_to_add=%*
+  set current_path_comparable=%path%
+  set current_path_comparable=%current_path_comparable: =_%
+  set current_path_comparable=%current_path_comparable:(=_%
+  set current_path_comparable=%current_path_comparable:)=_%
+  set path_to_add_comparable=%path_to_add%
+  set path_to_add_comparable=%path_to_add_comparable: =_%
+  set path_to_add_comparable=%path_to_add_comparable:(=_%
+  set path_to_add_comparable=%path_to_add_comparable:)=_%
+
+  for %%i in ( %current_path_comparable% ) do (
+    if /i "%%i" == "%path_to_add_comparable%" (
+      set path_to_add_exist=true
+    )
+  )
+  set system_path_comparable=
+  set path_to_add_comparable=
+  if not defined path_to_add_exist path=%path_to_add%;%path%
+  set path_to_add=
+  goto :eof
+
+@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
+:make_command_arguments
+  if "%1" == "--config" (
+    shift
+    shift
+  )
+  if [%2] == [] goto :eof
+  shift
+  set _arguments=
+  :MakeCmdArgsLoop 
+  if [%1]==[] goto :EndLoop 
+
+  if not defined _arguments (
+    set _arguments=%1
+  ) else (
+    set _arguments=!_arguments! %1
+  )
+  shift
+  goto :MakeCmdArgsLoop 
+  :EndLoop 
+  set hadoop-command-arguments=%_arguments%
+  goto :eof
+
+:print_usage
+  @echo Usage: hadoop [--config confdir] COMMAND
+  @echo where COMMAND is one of:
+  @echo   fs                   run a generic filesystem user client
+  @echo   version              print the version
+  @echo   jar ^<jar^>            run a jar file
+  @echo   distcp ^<srcurl^> ^<desturl^> copy file or directories recursively
+  @echo   archive -archiveName NAME -p ^<parent path^> ^<src^>* ^<dest^> create a hadoop archive
+  @echo   classpath            prints the class path needed to get the
+  @echo                        Hadoop jar and the required libraries
+  @echo   daemonlog            get/set the log level for each daemon
+  @echo  or
+  @echo   CLASSNAME            run the class named CLASSNAME
+  @echo.
+  @echo Most commands print help when invoked w/o parameters.
+
+endlocal

hadoop-common-project/hadoop-common/src/main/bin/rcc

@@ -57,10 +57,5 @@ unset IFS
 
 CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
 
-# cygwin path translation
-if expr `uname` : 'CYGWIN*' > /dev/null; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
 # run it
 exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"

hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd

@@ -0,0 +1,52 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+setlocal enabledelayedexpansion
+
+@rem Start all hadoop daemons.  Run this on master node.
+
+echo This script is Deprecated. Instead use start-dfs.cmd and start-yarn.cmd
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+  set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+if "%1" == "--config" (
+  shift
+  shift
+)
+
+@rem start hdfs daemons if hdfs is present
+if exist %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd (
+  call %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR%
+)
+
+@rem start yarn daemons if yarn is present
+if exist %HADOOP_YARN_HOME%\sbin\start-yarn.cmd (
+  call %HADOOP_YARN_HOME%\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR%
+)
+
+endlocal

hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd

@@ -0,0 +1,52 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+setlocal enabledelayedexpansion
+
+@rem Stop all hadoop daemons.  Run this on master node.
+
+echo This script is Deprecated. Instead use stop-dfs.cmd and stop-yarn.cmd
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+  set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+if "%1" == "--config" (
+  shift
+  shift
+)
+
+@rem stop hdfs daemons if hdfs is present
+if exist %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd (
+  call %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd --config %HADOOP_CONF_DIR%
+)
+
+@rem stop yarn daemons if yarn is present
+if exist %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd (
+  call %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd --config %HADOOP_CONF_DIR%
+)
+
+endlocal

hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd

@@ -0,0 +1,81 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set Hadoop-specific environment variables here.
+
+@rem The only required environment variable is JAVA_HOME.  All others are
+@rem optional.  When running a distributed configuration it is best to
+@rem set JAVA_HOME in this file, so that it is correctly defined on
+@rem remote nodes.
+
+@rem The java implementation to use.  Required.
+set JAVA_HOME=%JAVA_HOME%
+
+@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
+@rem set JSVC_HOME=%JSVC_HOME%
+
+@rem set HADOOP_CONF_DIR=
+
+@rem Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
+if exist %HADOOP_HOME%\contrib\capacity-scheduler (
+  if not defined HADOOP_CLASSPATH (
+    set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+  ) else (
+    set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+  )
+)
+
+@rem The maximum amount of heap to use, in MB. Default is 1000.
+@rem set HADOOP_HEAPSIZE=
+@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+@rem Extra Java runtime options.  Empty by default.
+@rem set HADOOP_OPTS=-Djava.net.preferIPv4Stack=true %HADOOP_CLIENT_OPTS%
+
+@rem Command specific options appended to HADOOP_OPTS when specified
+if not defined HADOOP_SECURITY_LOGGER (
+  set HADOOP_SECURITY_LOGGER=INFO,RFAS
+)
+if not defined HDFS_AUDIT_LOGGER (
+  set HDFS_AUDIT_LOGGER=INFO,NullAppender
+)
+
+set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
+set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
+set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
+
+@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
+@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
+
+@rem On secure datanodes, user to run the datanode as after dropping privileges
+set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
+
+@rem Where log files are stored.  %HADOOP_HOME%/logs by default.
+@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
+
+@rem Where log files are stored in the secure data environment.
+set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+
+@rem The directory where pid files are stored. /tmp by default.
+@rem NOTE: this should be set to a directory that can only be written to by 
+@rem       the user that will run the hadoop daemons.  Otherwise there is the
+@rem       potential for a symlink attack.
+set HADOOP_PID_DIR=%HADOOP_PID_DIR%
+set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
+
+@rem A string representing this instance of hadoop. %USERNAME% by default.
+set HADOOP_IDENT_STRING=%USERNAME%

hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml

@@ -77,7 +77,7 @@
 
  <property>
     <name>security.admin.operations.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for AdminOperationsProtocol. Used for admin commands.
     The ACL is a comma-separated list of user and group names. The user and
     group list is separated by a blank. For e.g. "alice,bob users,wheel".
@@ -86,7 +86,7 @@
 
   <property>
     <name>security.refresh.usertogroups.mappings.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for RefreshUserMappingsProtocol. Used to refresh
     users mappings. The ACL is a comma-separated list of user and
     group names. The user and group list is separated by a blank. For
@@ -96,7 +96,7 @@
 
   <property>
     <name>security.refresh.policy.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
     dfsadmin and mradmin commands to refresh the security policy in-effect.
     The ACL is a comma-separated list of user and group names. The user and
@@ -120,7 +120,7 @@
 
   <property>
     <name>security.qjournal.service.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for QJournalProtocol, used by the NN to communicate with
     JNs when using the QuorumJournalManager for edit logs.</description>
   </property>
@@ -139,7 +139,7 @@
 
   <property>
     <name>security.resourcetracker.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
+    <value>*</value>
     <description>ACL for ResourceTracker protocol, used by the
     ResourceManager and NodeManager to communicate with each other.
     The ACL is a comma-separated list of user and group names. The user and
@@ -149,7 +149,7 @@
 
   <property>
     <name>security.admin.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
+    <value>*</value>
     <description>ACL for RMAdminProtocol, for admin commands. 
     The ACL is a comma-separated list of user and group names. The user and
     group list is separated by a blank. For e.g. "alice,bob users,wheel".

hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml

@@ -87,7 +87,6 @@ See http://forrest.apache.org/docs/linking.html for more info.
     <zlib      href="http://www.zlib.net/" />
     <gzip      href="http://www.gzip.org/" />
     <bzip      href="http://www.bzip.org/" />
-    <cygwin    href="http://www.cygwin.com/" />
     <osx       href="http://www.apple.com/macosx" />
     
     <relnotes href="releasenotes.html" />

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java

@@ -21,8 +21,6 @@ package org.apache.hadoop.fs;
 import java.io.*;
 import java.util.Arrays;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -32,7 +30,7 @@ import org.apache.hadoop.util.PureJavaCrc32;
 
 /****************************************************************
  * Abstract Checksumed FileSystem.
- * It provide a basice implementation of a Checksumed FileSystem,
+ * It provide a basic implementation of a Checksumed FileSystem,
  * which creates a checksum file for each raw file.
  * It generates & verifies checksums at the client side.
  *
@@ -118,9 +116,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
    * It verifies that data matches checksums.
    *******************************************************/
   private static class ChecksumFSInputChecker extends FSInputChecker {
-    public static final Log LOG 
-      = LogFactory.getLog(FSInputChecker.class);
-    
     private ChecksumFileSystem fs;
     private FSDataInputStream datas;
     private FSDataInputStream sums;
@@ -374,19 +369,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
     private FSDataOutputStream sums;
     private static final float CHKSUM_AS_FRACTION = 0.01f;
     
-    public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
-                          Path file, 
-                          boolean overwrite, 
-                          short replication,
-                          long blockSize,
-                          Configuration conf)
-      throws IOException {
-      this(fs, file, overwrite, 
-           conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
-		       LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT),
-           replication, blockSize, null);
-    }
-    
     public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
                           Path file, 
                           boolean overwrite,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java

@@ -17,22 +17,25 @@
  */
 package org.apache.hadoop.fs;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.BufferedReader;
-
+import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.NoSuchElementException;
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.util.Shell;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /** Filesystem disk space usage statistics.
  * Uses the unix 'df' program to get mount points, and java.io.File for
- * space utilization. Tested on Linux, FreeBSD, Cygwin. */
+ * space utilization. Tested on Linux, FreeBSD, Windows. */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public class DF extends Shell {
@@ -44,37 +47,8 @@ public class DF extends Shell {
   private final File dirFile;
   private String filesystem;
   private String mount;
-
-  enum OSType {
-    OS_TYPE_UNIX("UNIX"),
-    OS_TYPE_WIN("Windows"),
-    OS_TYPE_SOLARIS("SunOS"),
-    OS_TYPE_MAC("Mac"),
-    OS_TYPE_AIX("AIX");
-
-    private String id;
-    OSType(String id) {
-      this.id = id;
-    }
-    public boolean match(String osStr) {
-      return osStr != null && osStr.indexOf(id) >= 0;
-    }
-    String getId() {
-      return id;
-    }
-  }
-
-  private static final String OS_NAME = System.getProperty("os.name");
-  private static final OSType OS_TYPE = getOSType(OS_NAME);
-
-  protected static OSType getOSType(String osName) {
-    for (OSType ost : EnumSet.allOf(OSType.class)) {
-      if (ost.match(osName)) {
-        return ost;
-      }
-    }
-    return OSType.OS_TYPE_UNIX;
-  }
+  
+  private ArrayList<String> output;
 
   public DF(File path, Configuration conf) throws IOException {
     this(path, conf.getLong(CommonConfigurationKeys.FS_DF_INTERVAL_KEY, DF.DF_INTERVAL_DEFAULT));
@@ -84,12 +58,9 @@ public class DF extends Shell {
     super(dfInterval);
     this.dirPath = path.getCanonicalPath();
     this.dirFile = new File(this.dirPath);
+    this.output = new ArrayList<String>();
   }
 
-  protected OSType getOSType() {
-    return OS_TYPE;
-  }
-  
   /// ACCESSORS
 
   /** @return the canonical path to the volume we're checking. */
@@ -99,8 +70,13 @@ public class DF extends Shell {
 
   /** @return a string indicating which filesystem volume we're checking. */
   public String getFilesystem() throws IOException {
-    run();
-    return filesystem;
+    if (Shell.WINDOWS) {
+      this.filesystem = dirFile.getCanonicalPath().substring(0, 2);
+      return this.filesystem;
+    } else {
+      run();
+      return filesystem;
+    }
   }
 
   /** @return the capacity of the measured filesystem in bytes. */
@@ -127,7 +103,28 @@ public class DF extends Shell {
 
   /** @return the filesystem mount point for the indicated volume */
   public String getMount() throws IOException {
-    run();
+    // Abort early if specified path does not exist
+    if (!dirFile.exists()) {
+      throw new FileNotFoundException("Specified path " + dirFile.getPath()
+          + "does not exist");
+    }
+
+    if (Shell.WINDOWS) {
+      // Assume a drive letter for a mount point
+      this.mount = dirFile.getCanonicalPath().substring(0, 2);
+    } else {
+      run();
+      // Skip parsing if df was not successful
+      if (getExitCode() != 0) {
+        StringBuffer sb = new StringBuffer("df could not be run successfully: ");
+        for (String line: output) {
+          sb.append(line);
+        }
+        throw new IOException(sb.toString());
+      }
+      parseOutput();
+    }
+
     return mount;
   }
   
@@ -146,52 +143,65 @@ public class DF extends Shell {
   @Override
   protected String[] getExecString() {
     // ignoring the error since the exit code it enough
-    return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath 
-                         + "' 2>/dev/null"};
+    if (Shell.WINDOWS){
+      throw new AssertionError(
+          "DF.getExecString() should never be called on Windows");
+    } else {
+      return new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath 
+                      + "' 2>/dev/null"};
+    }
   }
 
   @Override
   protected void parseExecResult(BufferedReader lines) throws IOException {
-    lines.readLine();                         // skip headings
-  
+    output.clear();
     String line = lines.readLine();
-    if (line == null) {
-      throw new IOException( "Expecting a line not the end of stream" );
+    while (line != null) {
+      output.add(line);
+      line = lines.readLine();
     }
+  }
+  
+  @VisibleForTesting
+  protected void parseOutput() throws IOException {
+    if (output.size() < 2) {
+      StringBuffer sb = new StringBuffer("Fewer lines of output than expected");
+      if (output.size() > 0) {
+        sb.append(": " + output.get(0));
+      }
+      throw new IOException(sb.toString());
+    }
+    
+    String line = output.get(1);
     StringTokenizer tokens =
       new StringTokenizer(line, " \t\n\r\f%");
     
-    this.filesystem = tokens.nextToken();
+    try {
+      this.filesystem = tokens.nextToken();
+    } catch (NoSuchElementException e) {
+      throw new IOException("Unexpected empty line");
+    }
     if (!tokens.hasMoreTokens()) {            // for long filesystem name
-      line = lines.readLine();
-      if (line == null) {
-        throw new IOException( "Expecting a line not the end of stream" );
+      if (output.size() > 2) {
+        line = output.get(2);
+      } else {
+        throw new IOException("Expecting additional output after line: "
+            + line);
       }
       tokens = new StringTokenizer(line, " \t\n\r\f%");
     }
 
-    switch(getOSType()) {
-      case OS_TYPE_AIX:
-        Long.parseLong(tokens.nextToken()); // capacity
-        Long.parseLong(tokens.nextToken()); // available
-        Integer.parseInt(tokens.nextToken()); // pct used
-        tokens.nextToken();
-        tokens.nextToken();
-        this.mount = tokens.nextToken();
-        break;
-
-      case OS_TYPE_WIN:
-      case OS_TYPE_SOLARIS:
-      case OS_TYPE_MAC:
-      case OS_TYPE_UNIX:
-      default:
-        Long.parseLong(tokens.nextToken()); // capacity
-        Long.parseLong(tokens.nextToken()); // used
-        Long.parseLong(tokens.nextToken()); // available
-        Integer.parseInt(tokens.nextToken()); // pct used
-        this.mount = tokens.nextToken();
-        break;
-   }
+    try {
+      Long.parseLong(tokens.nextToken()); // capacity
+      Long.parseLong(tokens.nextToken()); // used
+      Long.parseLong(tokens.nextToken()); // available
+      Integer.parseInt(tokens.nextToken()); // pct used
+      this.mount = tokens.nextToken();
+    } catch (NoSuchElementException e) {
+      throw new IOException("Could not parse line: " + line);
+    } catch (NumberFormatException e) {
+      throw new IOException("Could not parse line: " + line);
+    }
   }
 
   public static void main(String[] args) throws Exception {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java

@@ -145,6 +145,20 @@ public class DU extends Shell {
   public String getDirPath() {
     return dirPath;
   }
+
+
+  /**
+   * Override to hook in DUHelper class. Maybe this can be used more
+   * generally as well on Unix/Linux based systems
+   */
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      used.set(DUHelper.getFolderUsage(dirPath));
+      return;
+    }
+    super.run();
+  }
   
   /**
    * Start the disk usage checking thread.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java

@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.File;
+import org.apache.hadoop.util.Shell;
+
+public class DUHelper {
+
+  private int folderCount=0;
+  private int fileCount=0;
+  private double usage = 0;
+  private long folderSize = -1;
+
+  private DUHelper() {
+
+  }
+
+  public static long getFolderUsage(String folder) {
+    return new DUHelper().calculateFolderSize(folder);
+  }
+
+  private long calculateFolderSize(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+    return folderSize = getFileSize(f);
+  }
+
+  public String check(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+
+    folderSize = getFileSize(f);
+    usage = 1.0*(f.getTotalSpace() - f.getFreeSpace())/ f.getTotalSpace();
+    return String.format("used %d files %d disk in use %f", folderSize, fileCount, usage);
+  }
+
+  public long getFileCount() {
+    return fileCount;
+  }
+
+  public double getUsage() {
+    return usage;
+  }
+
+  private long getFileSize(File folder) {
+
+    folderCount++;
+    //Counting the total folders
+    long foldersize = 0;
+    if (folder.isFile())
+      return folder.length();
+    File[] filelist = folder.listFiles();
+    if (filelist == null) {
+      return 0;
+    }
+    for (int i = 0; i < filelist.length; i++) {
+      if (filelist[i].isDirectory()) {
+        foldersize += getFileSize(filelist[i]);
+      } else {
+        fileCount++; //Counting the total files
+        foldersize += filelist[i].length();
+      }
+    }
+    return foldersize;    
+  }
+
+  public static void main(String[] args) {
+    if (Shell.WINDOWS)
+      System.out.println("Windows: "+ DUHelper.getFolderUsage(args[0]));
+    else
+      System.out.println("Other: " + DUHelper.getFolderUsage(args[0]));
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java

@@ -244,17 +244,33 @@ public final class FileContext {
   }
  
   /* 
-   * Remove relative part - return "absolute":
-   * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
-   * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
+   * Resolve a relative path passed from the user.
+   * 
+   * Relative paths are resolved against the current working directory
+   * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
+   * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
    * ("/foo/bar") are returned unchanged.
    * 
+   * Additionally, we fix malformed URIs that specify a scheme but not an 
+   * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
+   * if it matches the default FS, and let the default FS add in the default
+   * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
+   * 
    * Applications that use FileContext should use #makeQualified() since
-   * they really want a fully qualified URI.
+   * they really want a fully-qualified URI.
    * Hence this method is not called makeAbsolute() and 
    * has been deliberately declared private.
    */
   private Path fixRelativePart(Path p) {
+    // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
+    if (p.hasSchemeAndNoAuthority()) {
+      String scheme = p.toUri().getScheme();
+      if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
+        p = new Path(p.toUri().getSchemeSpecificPart());
+      }
+    }
+    // Absolute paths are unchanged. Relative paths are resolved against the
+    // current working directory.
     if (p.isUriPathAbsolute()) {
       return p;
     } else {
@@ -1326,7 +1342,7 @@ public final class FileContext {
    * 
    * 2. Partially qualified URIs (eg scheme but no host)
    * 
-   * fs:///A/B/file  Resolved according to the target file sytem. Eg resolving
+   * fs:///A/B/file  Resolved according to the target file system. Eg resolving
    *                 a symlink to hdfs:///A results in an exception because
    *                 HDFS URIs must be fully qualified, while a symlink to 
    *                 file:///A will not since Hadoop's local file systems 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -21,6 +21,7 @@ import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -211,12 +212,46 @@ public abstract class FileSystem extends Configured implements Closeable {
   public abstract URI getUri();
   
   /**
-   * Resolve the uri's hostname and add the default port if not in the uri
+   * Return a canonicalized form of this FileSystem's URI.
+   * 
+   * The default implementation simply calls {@link #canonicalizeUri(URI)}
+   * on the filesystem's own URI, so subclasses typically only need to
+   * implement that method.
+   *
+   * @see #canonicalizeUri(URI)
+   */
+  protected URI getCanonicalUri() {
+    return canonicalizeUri(getUri());
+  }
+  
+  /**
+   * Canonicalize the given URI.
+   * 
+   * This is filesystem-dependent, but may for example consist of
+   * canonicalizing the hostname using DNS and adding the default
+   * port if not specified.
+   * 
+   * The default implementation simply fills in the default port if
+   * not specified and if the filesystem has a default port.
+   *
    * @return URI
    * @see NetUtils#getCanonicalUri(URI, int)
    */
-  protected URI getCanonicalUri() {
-    return NetUtils.getCanonicalUri(getUri(), getDefaultPort());
+  protected URI canonicalizeUri(URI uri) {
+    if (uri.getPort() == -1 && getDefaultPort() > 0) {
+      // reconstruct the uri with the default port set
+      try {
+        uri = new URI(uri.getScheme(), uri.getUserInfo(),
+            uri.getHost(), getDefaultPort(),
+            uri.getPath(), uri.getQuery(), uri.getFragment());
+      } catch (URISyntaxException e) {
+        // Should never happen!
+        throw new AssertionError("Valid URI became unparseable: " +
+            uri);
+      }
+    }
+    
+    return uri;
   }
   
   /**
@@ -581,7 +616,7 @@ public abstract class FileSystem extends Configured implements Closeable {
       }
       if (uri != null) {
         // canonicalize uri before comparing with this fs
-        uri = NetUtils.getCanonicalUri(uri, getDefaultPort());
+        uri = canonicalizeUri(uri);
         thatAuthority = uri.getAuthority();
         if (thisAuthority == thatAuthority ||       // authorities match
             (thisAuthority != null &&
@@ -1864,7 +1899,7 @@ public abstract class FileSystem extends Configured implements Closeable {
    * 
    * Some file systems like LocalFileSystem have an initial workingDir
    * that we use as the starting workingDir. For other file systems
-   * like HDFS there is no built in notion of an inital workingDir.
+   * like HDFS there is no built in notion of an initial workingDir.
    * 
    * @return if there is built in notion of workingDir then it
    * is returned; else a null is returned.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java

@@ -19,18 +19,29 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import org.apache.commons.collections.map.CaseInsensitiveMap;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
@@ -46,6 +57,13 @@ public class FileUtil {
 
   private static final Log LOG = LogFactory.getLog(FileUtil.class);
 
+  /* The error code is defined in winutils to indicate insufficient
+   * privilege to create symbolic links. This value need to keep in
+   * sync with the constant of the same name in:
+   * "src\winutils\common.h"
+   * */
+  public static final int SYMLINK_NO_PRIVILEGE = 2;
+
   /**
    * convert an array of FileStatus to an array of Path
    * 
@@ -469,34 +487,6 @@ public class FileUtil {
     return dst;
   }
 
-  /**
-   * This class is only used on windows to invoke the cygpath command.
-   */
-  private static class CygPathCommand extends Shell {
-    String[] command;
-    String result;
-    CygPathCommand(String path) throws IOException {
-      command = new String[]{"cygpath", "-u", path};
-      run();
-    }
-    String getResult() throws IOException {
-      return result;
-    }
-    @Override
-    protected String[] getExecString() {
-      return command;
-    }
-    @Override
-    protected void parseExecResult(BufferedReader lines) throws IOException {
-      String line = lines.readLine();
-      if (line == null) {
-        throw new IOException("Can't convert '" + command[2] + 
-                              " to a cygwin path");
-      }
-      result = line;
-    }
-  }
-
   /**
    * Convert a os-native filename to a path that works for the shell.
    * @param filename The filename to convert
@@ -504,11 +494,7 @@ public class FileUtil {
    * @throws IOException on windows, there can be problems with the subprocess
    */
   public static String makeShellPath(String filename) throws IOException {
-    if (Path.WINDOWS) {
-      return new CygPathCommand(filename).getResult();
-    } else {
-      return filename;
-    }    
+    return filename;
   }
   
   /**
@@ -658,7 +644,7 @@ public class FileUtil {
     untarCommand.append(FileUtil.makeShellPath(untarDir)); 
     untarCommand.append("' ; ");
     untarCommand.append("tar -xf ");
-    
+
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
@@ -731,7 +717,7 @@ public class FileUtil {
   
   /**
    * Class for creating hardlinks.
-   * Supports Unix, Cygwin, WindXP.
+   * Supports Unix, WindXP.
    * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
    */
   @Deprecated
@@ -743,21 +729,67 @@ public class FileUtil {
 
   /**
    * Create a soft link between a src and destination
-   * only on a local disk. HDFS does not support this
+   * only on a local disk. HDFS does not support this.
+   * On Windows, when symlink creation fails due to security
+   * setting, we will log a warning. The return code in this
+   * case is 2.
    * @param target the target for symlink 
    * @param linkname the symlink
    * @return value returned by the command
    */
   public static int symLink(String target, String linkname) throws IOException{
-    String cmd = "ln -s " + target + " " + linkname;
-    Process p = Runtime.getRuntime().exec(cmd, null);
-    int returnVal = -1;
-    try{
-      returnVal = p.waitFor();
-    } catch(InterruptedException e){
-      //do nothing as of yet
+    // Run the input paths through Java's File so that they are converted to the
+    // native OS form
+    File targetFile = new File(target);
+    File linkFile = new File(linkname);
+
+    // If not on Java7+, copy a file instead of creating a symlink since
+    // Java6 has close to no support for symlinks on Windows. Specifically
+    // File#length and File#renameTo do not work as expected.
+    // (see HADOOP-9061 for additional details)
+    // We still create symlinks for directories, since the scenario in this
+    // case is different. The directory content could change in which
+    // case the symlink loses its purpose (for example task attempt log folder
+    // is symlinked under userlogs and userlogs are generated afterwards).
+    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
+      try {
+        LOG.info("FileUtil#symlink: On Java6, copying file instead "
+            + linkname + " -> " + target);
+        org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+      } catch (IOException ex) {
+        LOG.warn("FileUtil#symlink failed to copy the file with error: "
+            + ex.getMessage());
+        // Exit with non-zero exit code
+        return 1;
+      }
+      return 0;
     }
-    return returnVal;
+
+    String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
+        linkFile.getPath());
+    ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+    try {
+      shExec.execute();
+    } catch (Shell.ExitCodeException ec) {
+      int returnVal = ec.getExitCode();
+      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+        LOG.warn("Fail to create symbolic links on Windows. "
+            + "The default security settings in Windows disallow non-elevated "
+            + "administrators and all non-administrators from creating symbolic links. "
+            + "This behavior can be changed in the Local Security Policy management console");
+      } else if (returnVal != 0) {
+        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+            + returnVal + " with: " + ec.getMessage());
+      }
+      return returnVal;
+    } catch (IOException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error while create symlink " + linkname + " to " + target
+            + "." + " Exception: " + StringUtils.stringifyException(e));
+      }
+      throw e;
+    }
+    return shExec.getExitCode();
   }
   
   /**
@@ -781,30 +813,120 @@ public class FileUtil {
    * @param recursive true, if permissions should be changed recursively
    * @return the exit code from the command.
    * @throws IOException
-   * @throws InterruptedException
    */
   public static int chmod(String filename, String perm, boolean recursive)
-                            throws IOException, InterruptedException {
-    StringBuilder cmdBuf = new StringBuilder();
-    cmdBuf.append("chmod ");
-    if (recursive) {
-      cmdBuf.append("-R ");
-    }
-    cmdBuf.append(perm).append(" ");
-    cmdBuf.append(filename);
-    String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
-    ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+                            throws IOException {
+    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = new File(filename).getPath();
+    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
     try {
       shExec.execute();
-    }catch(Exception e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Error while changing permission : " + filename
-            + " Exception: ", e);
+    }catch(IOException e) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Error while changing permission : " + filename 
+                  +" Exception: " + StringUtils.stringifyException(e));
       }
     }
     return shExec.getExitCode();
   }
+
+  /**
+   * Set the ownership on a file / directory. User name and group name
+   * cannot both be null.
+   * @param file the file to change
+   * @param username the new user owner name
+   * @param groupname the new group owner name
+   * @throws IOException
+   */
+  public static void setOwner(File file, String username,
+      String groupname) throws IOException {
+    if (username == null && groupname == null) {
+      throw new IOException("username == null && groupname == null");
+    }
+    String arg = (username == null ? "" : username)
+        + (groupname == null ? "" : ":" + groupname);
+    String [] cmd = Shell.getSetOwnerCommand(arg);
+    execCommand(file, cmd);
+  }
+
+  /**
+   * Set permissions to the required value. Uses the java primitives instead
+   * of forking if group == other.
+   * @param f the file to change
+   * @param permission the new permissions
+   * @throws IOException
+   */
+  public static void setPermission(File f, FsPermission permission
+                                   ) throws IOException {
+    FsAction user = permission.getUserAction();
+    FsAction group = permission.getGroupAction();
+    FsAction other = permission.getOtherAction();
+
+    // use the native/fork if the group/other permissions are different
+    // or if the native is available or on Windows
+    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+      execSetPermission(f, permission);
+      return;
+    }
+    
+    boolean rv = true;
+    
+    // read perms
+    rv = f.setReadable(group.implies(FsAction.READ), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+      rv = f.setReadable(user.implies(FsAction.READ), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // write perms
+    rv = f.setWritable(group.implies(FsAction.WRITE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+      rv = f.setWritable(user.implies(FsAction.WRITE), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // exec perms
+    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+      rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
+      checkReturnValue(rv, f, permission);
+    }
+  }
+
+  private static void checkReturnValue(boolean rv, File p, 
+                                       FsPermission permission
+                                       ) throws IOException {
+    if (!rv) {
+      throw new IOException("Failed to set permissions of path: " + p + 
+                            " to " + 
+                            String.format("%04o", permission.toShort()));
+    }
+  }
+  
+  private static void execSetPermission(File f, 
+                                        FsPermission permission
+                                       )  throws IOException {
+    if (NativeIO.isAvailable()) {
+      NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
+    } else {
+      execCommand(f, Shell.getSetPermissionCommand(
+                  String.format("%04o", permission.toShort()), false));
+    }
+  }
   
+  static String execCommand(File f, String... cmd) throws IOException {
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = f.getCanonicalPath();
+    String output = Shell.execCommand(args);
+    return output;
+  }
+
   /**
    * Create a tmp file for a base file.
    * @param basefile the base file of the tmp
@@ -892,4 +1014,97 @@ public class FileUtil {
     }
     return fileNames;
   }  
+  
+  /**
+   * Create a jar file at the given path, containing a manifest with a classpath
+   * that references all specified entries.
+   * 
+   * Some platforms may have an upper limit on command line length.  For example,
+   * the maximum command line length on Windows is 8191 characters, but the
+   * length of the classpath may exceed this.  To work around this limitation,
+   * use this method to create a small intermediate jar with a manifest that
+   * contains the full classpath.  It returns the absolute path to the new jar,
+   * which the caller may set as the classpath for a new process.
+   * 
+   * Environment variable evaluation is not supported within a jar manifest, so
+   * this method expands environment variables before inserting classpath entries
+   * to the manifest.  The method parses environment variables according to
+   * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+   * environment variables are case-insensitive.  For example, %VAR% and %var%
+   * evaluate to the same value.
+   * 
+   * Specifying the classpath in a jar manifest does not support wildcards, so
+   * this method expands wildcards internally.  Any classpath entry that ends
+   * with * is translated to all files at that path with extension .jar or .JAR.
+   * 
+   * @param inputClassPath String input classpath to bundle into the jar manifest
+   * @param pwd Path to working directory to save jar
+   * @return String absolute path to new jar
+   * @throws IOException if there is an I/O error while writing the jar file
+   */
+  public static String createJarWithClassPath(String inputClassPath, Path pwd)
+      throws IOException {
+    // Replace environment variables, case-insensitive on Windows
+    @SuppressWarnings("unchecked")
+    Map<String, String> env = Shell.WINDOWS ?
+      new CaseInsensitiveMap(System.getenv()) : System.getenv();
+    String[] classPathEntries = inputClassPath.split(File.pathSeparator);
+    for (int i = 0; i < classPathEntries.length; ++i) {
+      classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
+        StringUtils.ENV_VAR_PATTERN, env);
+    }
+    File workingDir = new File(pwd.toString());
+    if (!workingDir.mkdirs()) {
+      // If mkdirs returns false because the working directory already exists,
+      // then this is acceptable.  If it returns false due to some other I/O
+      // error, then this method will fail later with an IOException while saving
+      // the jar.
+      LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
+    }
+
+    // Append all entries
+    List<String> classPathEntryList = new ArrayList<String>(
+      classPathEntries.length);
+    for (String classPathEntry: classPathEntries) {
+      if (classPathEntry.endsWith("*")) {
+        // Append all jars that match the wildcard
+        Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
+        FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
+          .globStatus(globPath);
+        if (wildcardJars != null) {
+          for (FileStatus wildcardJar: wildcardJars) {
+            classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
+              .toExternalForm());
+          }
+        }
+      } else {
+        // Append just this jar
+        classPathEntryList.add(new File(classPathEntry).toURI().toURL()
+          .toExternalForm());
+      }
+    }
+    String jarClassPath = StringUtils.join(" ", classPathEntryList);
+
+    // Create the manifest
+    Manifest jarManifest = new Manifest();
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.CLASS_PATH.toString(), jarClassPath);
+
+    // Write the manifest to output JAR file
+    File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
+    FileOutputStream fos = null;
+    BufferedOutputStream bos = null;
+    JarOutputStream jos = null;
+    try {
+      fos = new FileOutputStream(classPathJar);
+      bos = new BufferedOutputStream(fos);
+      jos = new JarOutputStream(bos, jarManifest);
+    } finally {
+      IOUtils.cleanup(LOG, jos, bos, fos);
+    }
+
+    return classPathJar.getCanonicalPath();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -95,16 +95,18 @@ public class FilterFileSystem extends FileSystem {
   public URI getUri() {
     return fs.getUri();
   }
-
-  /**
-   * Returns a qualified URI whose scheme and authority identify this
-   * FileSystem.
-   */
+  
+  
   @Override
   protected URI getCanonicalUri() {
     return fs.getCanonicalUri();
   }
-  
+
+  @Override
+  protected URI canonicalizeUri(URI uri) {
+    return fs.canonicalizeUri(uri);
+  }
+
   /** Make sure that a path specifies a FileSystem. */
   @Override
   public Path makeQualified(Path path) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java

@@ -25,9 +25,11 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Arrays;
 
+import org.apache.hadoop.util.Shell;
+
 /**
  * Class for creating hardlinks.
- * Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X.
+ * Supports Unix/Linux, Windows via winutils , and Mac OS X.
  * 
  * The HardLink class was formerly a static inner class of FSUtil,
  * and the methods provided were blatantly non-thread-safe.
@@ -41,7 +43,7 @@ public class HardLink {
 
   public enum OSType {
     OS_TYPE_UNIX,
-    OS_TYPE_WINXP,
+    OS_TYPE_WIN,
     OS_TYPE_SOLARIS,
     OS_TYPE_MAC,
     OS_TYPE_FREEBSD
@@ -56,7 +58,7 @@ public class HardLink {
   //methods without instantiating the HardLink object
   static { 
     osType = getOSType();
-    if (osType == OSType.OS_TYPE_WINXP) {
+    if (osType == OSType.OS_TYPE_WIN) {
       // Windows
       getHardLinkCommand = new HardLinkCGWin();
     } else {
@@ -80,14 +82,8 @@ public class HardLink {
   
   static private OSType getOSType() {
     String osName = System.getProperty("os.name");
-    if (osName.contains("Windows") &&
-            (osName.contains("XP") 
-            || osName.contains("2003") 
-            || osName.contains("Vista")
-            || osName.contains("Windows_7")
-            || osName.contains("Windows 7") 
-            || osName.contains("Windows7"))) {
-      return OSType.OS_TYPE_WINXP;
+    if (Shell.WINDOWS) {
+      return OSType.OS_TYPE_WIN;
     }
     else if (osName.contains("SunOS") 
             || osName.contains("Solaris")) {
@@ -258,11 +254,6 @@ public class HardLink {
   
   /**
    * Implementation of HardLinkCommandGetter class for Windows
-   * 
-   * Note that the linkCount shell command for Windows is actually
-   * a Cygwin shell command, and depends on ${cygwin}/bin
-   * being in the Windows PATH environment variable, so
-   * stat.exe can be found.
    */
   static class HardLinkCGWin extends HardLinkCommandGetter {
     //The Windows command getter impl class and its member fields are
@@ -270,14 +261,16 @@ public class HardLink {
     //unit testing (sort of) on non-Win servers
 
     static String[] hardLinkCommand = {
-                        "fsutil","hardlink","create", null, null};
+                        Shell.WINUTILS,"hardlink","create", null, null};
     static String[] hardLinkMultPrefix = {
                         "cmd","/q","/c","for", "%f", "in", "("};
     static String   hardLinkMultDir = "\\%f";
     static String[] hardLinkMultSuffix = {
-                        ")", "do", "fsutil", "hardlink", "create", null, 
+                        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
                         "%f", "1>NUL"};
-    static String[] getLinkCountCommand = {"stat","-c%h", null};
+    static String[] getLinkCountCommand = {
+                        Shell.WINUTILS, "hardlink",
+                        "stat", null};
     //Windows guarantees only 8K - 1 bytes cmd length.
     //Subtract another 64b to allow for Java 'exec' overhead
     static final int maxAllowedCmdArgLength = 8*1024 - 65;
@@ -328,12 +321,6 @@ public class HardLink {
       String[] buf = new String[getLinkCountCommand.length];
       System.arraycopy(getLinkCountCommand, 0, buf, 0, 
                        getLinkCountCommand.length);
-      //The linkCount command is actually a Cygwin shell command,
-      //not a Windows shell command, so we should use "makeShellPath()"
-      //instead of "getCanonicalPath()".  However, that causes another
-      //shell exec to "cygpath.exe", and "stat.exe" actually can handle
-      //DOS-style paths (it just prints a couple hundred bytes of warning
-      //to stderr), so we use the more efficient "getCanonicalPath()".
       buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
       return buf;
     }
@@ -354,7 +341,7 @@ public class HardLink {
       //add the fixed overhead of the hardLinkMult command 
       //(prefix, suffix, and Dir suffix)
       sum += ("cmd.exe /q /c for %f in ( ) do "
-              + "fsutil hardlink create \\%f %f 1>NUL ").length();
+              + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length();
       return sum;
     }
     
@@ -581,14 +568,10 @@ public class HardLink {
   /* Create an IOException for failing to get link count. */
   private static IOException createIOException(File f, String message,
       String error, int exitvalue, Exception cause) {
-    
-    final String winErrMsg = "; Windows errors in getLinkCount are often due "
-         + "to Cygwin misconfiguration";
 
     final String s = "Failed to get link count on file " + f
         + ": message=" + message
         + "; error=" + error
-        + ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "")
         + "; exit value=" + exitvalue;
     return (cause == null) ? new IOException(s) : new IOException(s, cause);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.regex.Pattern;
 
 import org.apache.avro.reflect.Stringable;
 import org.apache.commons.lang.StringUtils;
@@ -43,9 +44,17 @@ public class Path implements Comparable {
   
   public static final String CUR_DIR = ".";
   
-  static final boolean WINDOWS
+  public static final boolean WINDOWS
     = System.getProperty("os.name").startsWith("Windows");
 
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern hasUriScheme =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern hasDriveLetterSpecifier =
+      Pattern.compile("^/?[a-zA-Z]:");
+
   private URI uri;                                // a hierarchical uri
 
   /** Resolve a child path against a parent path. */
@@ -81,7 +90,7 @@ public class Path implements Comparable {
                resolved.getPath(), resolved.getFragment());
   }
 
-  private void checkPathArg( String path ) {
+  private void checkPathArg( String path ) throws IllegalArgumentException {
     // disallow construction of a Path from an empty string
     if ( path == null ) {
       throw new IllegalArgumentException(
@@ -95,15 +104,16 @@ public class Path implements Comparable {
   
   /** Construct a path from a String.  Path strings are URIs, but with
    * unescaped elements and some additional normalization. */
-  public Path(String pathString) {
+  public Path(String pathString) throws IllegalArgumentException {
     checkPathArg( pathString );
     
     // We can't use 'new URI(String)' directly, since it assumes things are
     // escaped, which we don't require of Paths. 
     
     // add a slash in front of paths with Windows drive letters
-    if (hasWindowsDrive(pathString, false))
-      pathString = "/"+pathString;
+    if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') {
+      pathString = "/" + pathString;
+    }
 
     // parse uri components
     String scheme = null;
@@ -151,22 +161,54 @@ public class Path implements Comparable {
   private void initialize(String scheme, String authority, String path,
       String fragment) {
     try {
-      this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
+      this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment)
         .normalize();
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }
   }
 
-  private String normalizePath(String path) {
-    // remove double slashes & backslashes
+  /**
+   * Merge 2 paths such that the second path is appended relative to the first.
+   * The returned path has the scheme and authority of the first path.  On
+   * Windows, the drive specification in the second path is discarded.
+   * 
+   * @param path1 Path first path
+   * @param path2 Path second path, to be appended relative to path1
+   * @return Path merged path
+   */
+  public static Path mergePaths(Path path1, Path path2) {
+    String path2Str = path2.toUri().getPath();
+    if(hasWindowsDrive(path2Str)) {
+      path2Str = path2Str.substring(path2Str.indexOf(':')+1);
+    }
+    return new Path(path1 + path2Str);
+  }
+
+  /**
+   * Normalize a path string to use non-duplicated forward slashes as
+   * the path separator and remove any trailing path separators.
+   * @param scheme Supplies the URI scheme. Used to deduce whether we
+   *               should replace backslashes or not.
+   * @param path Supplies the scheme-specific part
+   * @return Normalized path string.
+   */
+  private static String normalizePath(String scheme, String path) {
+    // Remove double forward slashes.
     path = StringUtils.replace(path, "//", "/");
-    if (Path.WINDOWS) {
+
+    // Remove backslashes if this looks like a Windows path. Avoid
+    // the substitution if it looks like a non-local URI.
+    if (WINDOWS &&
+        (hasWindowsDrive(path) ||
+         (scheme == null) ||
+         (scheme.isEmpty()) ||
+         (scheme.equals("file")))) {
       path = StringUtils.replace(path, "\\", "/");
     }
     
     // trim trailing slash from non-root path (ignoring windows drive)
-    int minLength = hasWindowsDrive(path, true) ? 4 : 1;
+    int minLength = hasWindowsDrive(path) ? 4 : 1;
     if (path.length() > minLength && path.endsWith("/")) {
       path = path.substring(0, path.length()-1);
     }
@@ -174,17 +216,29 @@ public class Path implements Comparable {
     return path;
   }
 
-  private boolean hasWindowsDrive(String path, boolean slashed) {
-    if (!WINDOWS) return false;
-    int start = slashed ? 1 : 0;
-    return
-      path.length() >= start+2 &&
-      (slashed ? path.charAt(0) == '/' : true) &&
-      path.charAt(start+1) == ':' &&
-      ((path.charAt(start) >= 'A' && path.charAt(start) <= 'Z') ||
-       (path.charAt(start) >= 'a' && path.charAt(start) <= 'z'));
+  private static boolean hasWindowsDrive(String path) {
+    return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find());
   }
 
+  /**
+   * Determine whether a given path string represents an absolute path on
+   * Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+   *
+   * @param pathString Supplies the path string to evaluate.
+   * @param slashed true if the given path is prefixed with "/".
+   * @return true if the supplied path looks like an absolute path with a Windows
+   * drive-specifier.
+   */
+  public static boolean isWindowsAbsolutePath(final String pathString,
+                                              final boolean slashed) {
+    int start = (slashed ? 1 : 0);
+
+    return
+        hasWindowsDrive(pathString) &&
+        pathString.length() >= (start + 3) &&
+        ((pathString.charAt(start + 2) == SEPARATOR_CHAR) ||
+          (pathString.charAt(start + 2) == '\\'));
+  }
 
   /** Convert this to a URI. */
   public URI toUri() { return uri; }
@@ -202,12 +256,16 @@ public class Path implements Comparable {
     return  (isUriPathAbsolute() && 
         uri.getScheme() == null && uri.getAuthority() == null);
   }
+
+  public boolean hasSchemeAndNoAuthority() {
+    return uri.getScheme() != null && uri.getAuthority() == null;
+  }
   
   /**
    *  True if the path component (i.e. directory) of this URI is absolute.
    */
   public boolean isUriPathAbsolute() {
-    int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0;
+    int start = hasWindowsDrive(uri.getPath()) ? 3 : 0;
     return uri.getPath().startsWith(SEPARATOR, start);
    }
   
@@ -241,7 +299,7 @@ public class Path implements Comparable {
   public Path getParent() {
     String path = uri.getPath();
     int lastSlash = path.lastIndexOf('/');
-    int start = hasWindowsDrive(path, true) ? 3 : 0;
+    int start = hasWindowsDrive(path) ? 3 : 0;
     if ((path.length() == start) ||               // empty path
         (lastSlash == start && path.length() == start+1)) { // at root
       return null;
@@ -250,7 +308,7 @@ public class Path implements Comparable {
     if (lastSlash==-1) {
       parent = CUR_DIR;
     } else {
-      int end = hasWindowsDrive(path, true) ? 3 : 0;
+      int end = hasWindowsDrive(path) ? 3 : 0;
       parent = path.substring(0, lastSlash==end?end+1:lastSlash);
     }
     return new Path(uri.getScheme(), uri.getAuthority(), parent);
@@ -277,7 +335,7 @@ public class Path implements Comparable {
     if (uri.getPath() != null) {
       String path = uri.getPath();
       if (path.indexOf('/')==0 &&
-          hasWindowsDrive(path, true) &&          // has windows drive
+          hasWindowsDrive(path) &&                // has windows drive
           uri.getScheme() == null &&              // but no scheme
           uri.getAuthority() == null)             // or authority
         path = path.substring(1);                 // remove slash before drive
@@ -364,7 +422,7 @@ public class Path implements Comparable {
     URI newUri = null;
     try {
       newUri = new URI(scheme, authority , 
-        normalizePath(pathUri.getPath()), null, fragment);
+        normalizePath(scheme, pathUri.getPath()), null, fragment);
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java

@@ -43,7 +43,7 @@ public interface PositionedReadable {
     throws IOException;
   
   /**
-   * Read number of bytes equalt to the length of the buffer, from a given
+   * Read number of bytes equal to the length of the buffer, from a given
    * position within a file. This does not
    * change the current offset of a file, and is thread-safe.
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -508,9 +508,10 @@ public class RawLocalFileSystem extends FileSystem {
       return !super.getOwner().isEmpty(); 
     }
     
-    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
+    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
-            f.lastModified(), fs.makeQualified(new Path(f.getPath())));
+          f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
+            fs.getWorkingDirectory()));
     }
     
     @Override
@@ -541,9 +542,10 @@ public class RawLocalFileSystem extends FileSystem {
     private void loadPermissionInfo() {
       IOException e = null;
       try {
-        StringTokenizer t = new StringTokenizer(
-            execCommand(new File(getPath().toUri()), 
-                        Shell.getGET_PERMISSION_COMMAND()));
+        String output = FileUtil.execCommand(new File(getPath().toUri()), 
+            Shell.getGetPermissionCommand());
+        StringTokenizer t =
+            new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -552,7 +554,17 @@ public class RawLocalFileSystem extends FileSystem {
         }
         setPermission(FsPermission.valueOf(permission));
         t.nextToken();
-        setOwner(t.nextToken());
+
+        String owner = t.nextToken();
+        // If on windows domain, token format is DOMAIN\\user and we want to
+        // extract only the user name
+        if (Shell.WINDOWS) {
+          int i = owner.indexOf('\\');
+          if (i != -1)
+            owner = owner.substring(i + 1);
+        }
+        setOwner(owner);
+
         setGroup(t.nextToken());
       } catch (Shell.ExitCodeException ioe) {
         if (ioe.getExitCode() != 1) {
@@ -588,17 +600,7 @@ public class RawLocalFileSystem extends FileSystem {
   @Override
   public void setOwner(Path p, String username, String groupname)
     throws IOException {
-    if (username == null && groupname == null) {
-      throw new IOException("username == null && groupname == null");
-    }
-
-    if (username == null) {
-      execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
-    } else {
-      //OWNER[:[GROUP]]
-      String s = username + (groupname == null? "": ":" + groupname);
-      execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
-    }
+    FileUtil.setOwner(pathToFile(p), username, groupname);
   }
 
   /**
@@ -608,20 +610,12 @@ public class RawLocalFileSystem extends FileSystem {
   public void setPermission(Path p, FsPermission permission)
     throws IOException {
     if (NativeIO.isAvailable()) {
-      NativeIO.chmod(pathToFile(p).getCanonicalPath(),
+      NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                      permission.toShort());
     } else {
-      execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
-          String.format("%05o", permission.toShort()));
+      String perm = String.format("%04o", permission.toShort());
+      Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+        FileUtil.makeShellPath(pathToFile(p), true)));
     }
   }
-
-  private static String execCommand(File f, String... cmd) throws IOException {
-    String[] args = new String[cmd.length + 1];
-    System.arraycopy(cmd, 0, args, 0, cmd.length);
-    args[cmd.length] = FileUtil.makeShellPath(f, true);
-    String output = Shell.execCommand(args);
-    return output;
-  }
-
-}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java

@@ -79,19 +79,17 @@ public abstract class TrashPolicy extends Configured {
 
   /**
    * Get an instance of the configured TrashPolicy based on the value 
-   * of the configuration paramater fs.trash.classname.
+   * of the configuration parameter fs.trash.classname.
    *
    * @param conf the configuration to be used
    * @param fs the file system to be used
    * @param home the home directory
    * @return an instance of TrashPolicy
    */
-  public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home)
-      throws IOException {
-    Class<? extends TrashPolicy> trashClass = conf.getClass("fs.trash.classname",
-                                                      TrashPolicyDefault.class,
-                                                      TrashPolicy.class);
-    TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf);
+  public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
+    Class<? extends TrashPolicy> trashClass = conf.getClass(
+        "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+    TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
     trash.initialize(conf, fs, home); // initialize TrashPolicy
     return trash;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java

@@ -92,7 +92,7 @@ public class TrashPolicyDefault extends TrashPolicy {
   }
   
   private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
-    return new Path(basePath + rmFilePath.toUri().getPath());
+    return Path.mergePaths(basePath, rmFilePath);
   }
 
   @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java

@@ -89,11 +89,9 @@ public class RawLocalFs extends DelegateToFileSystem {
     }
     // NB: Use createSymbolicLink in java.nio.file.Path once available
     try {
-      Shell.execCommand(Shell.LINK_COMMAND, "-s",
-                        new URI(target.toString()).getPath(),
-                        new URI(link.toString()).getPath());
-    } catch (URISyntaxException x) {
-      throw new IOException("Invalid symlink path: "+x.getMessage());
+      Shell.execCommand(Shell.getSymlinkCommand(
+        getPathWithoutSchemeAndAuthority(target),
+        getPathWithoutSchemeAndAuthority(link)));
     } catch (IOException x) {
       throw new IOException("Unable to create symlink: "+x.getMessage());
     }
@@ -176,4 +174,13 @@ public class RawLocalFs extends DelegateToFileSystem {
      */
     throw new AssertionError();
   }
+
+  private static String getPathWithoutSchemeAndAuthority(Path path) {
+    // This code depends on Path.toString() to remove the leading slash before
+    // the drive specification on Windows.
+    Path newPath = path.isUriPathAbsolute() ?
+      new Path(null, null, path.toUri().getPath()) :
+      path;
+    return newPath.toString();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java

@@ -137,9 +137,15 @@ class Jets3tFileSystemStore implements FileSystemStore {
 
   @Override
   public boolean inodeExists(Path path) throws IOException {
-    InputStream in = get(pathToKey(path), true);
+    String key = pathToKey(path);
+    InputStream in = get(key, true);
     if (in == null) {
-      return false;
+      if (isRoot(key)) {
+        storeINode(path, INode.DIRECTORY_INODE);
+        return true;
+      } else {
+        return false;
+      }
     }
     in.close();
     return true;
@@ -211,7 +217,13 @@ class Jets3tFileSystemStore implements FileSystemStore {
 
   @Override
   public INode retrieveINode(Path path) throws IOException {
-    return INode.deserialize(get(pathToKey(path), true));
+    String key = pathToKey(path);
+    InputStream in = get(key, true);
+    if (in == null && isRoot(key)) {
+      storeINode(path, INode.DIRECTORY_INODE);
+      return INode.DIRECTORY_INODE;
+    }
+    return INode.deserialize(in);
   }
 
   @Override
@@ -366,6 +378,10 @@ class Jets3tFileSystemStore implements FileSystemStore {
     return blockToKey(block.getId());
   }
 
+  private boolean isRoot(String key) {
+    return key.isEmpty() || key.equals("/");
+  }
+
   @Override
   public void purge() throws IOException {
     try {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java

@@ -252,32 +252,125 @@ public class S3FileSystem extends FileSystem {
   @Override
   public boolean rename(Path src, Path dst) throws IOException {
     Path absoluteSrc = makeAbsolute(src);
+    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
     INode srcINode = store.retrieveINode(absoluteSrc);
+    boolean debugEnabled = LOG.isDebugEnabled();
     if (srcINode == null) {
       // src path doesn't exist
+      if (debugEnabled) {
+        LOG.debug(debugPreamble + "returning false as src does not exist");
+      }
       return false; 
     }
+
     Path absoluteDst = makeAbsolute(dst);
-    INode dstINode = store.retrieveINode(absoluteDst);
-    if (dstINode != null && dstINode.isDirectory()) {
-      absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
-      dstINode = store.retrieveINode(absoluteDst);
-    }
-    if (dstINode != null) {
-      // dst path already exists - can't overwrite
-      return false;
-    }
+
+    //validate the parent dir of the destination
     Path dstParent = absoluteDst.getParent();
     if (dstParent != null) {
+      //if the dst parent is not root, make sure it exists
       INode dstParentINode = store.retrieveINode(dstParent);
-      if (dstParentINode == null || dstParentINode.isFile()) {
-        // dst parent doesn't exist or is a file
+      if (dstParentINode == null) {
+        // dst parent doesn't exist
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "returning false as dst parent does not exist");
+        }
+        return false;
+      }
+      if (dstParentINode.isFile()) {
+        // dst parent exists but is a file
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "returning false as dst parent exists and is a file");
+        }
         return false;
       }
     }
+
+    //get status of source
+    boolean srcIsFile = srcINode.isFile();
+
+    INode dstINode = store.retrieveINode(absoluteDst);
+    boolean destExists = dstINode != null;
+    boolean destIsDir = destExists && !dstINode.isFile();
+    if (srcIsFile) {
+
+      //source is a simple file
+      if (destExists) {
+        if (destIsDir) {
+          //outcome #1 dest exists and is dir -filename to subdir of dest
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying src file under dest dir to " + absoluteDst);
+          }
+          absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+        } else {
+          //outcome #2 dest it's a file: fail iff different from src
+          boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying file onto file, outcome is " + renamingOnToSelf);
+          }
+          return renamingOnToSelf;
+        }
+      } else {
+        // #3 dest does not exist: use dest as path for rename
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "copying file onto file");
+        }
+      }
+    } else {
+      //here the source exists and is a directory
+      // outcomes (given we know the parent dir exists if we get this far)
+      // #1 destination is a file: fail
+      // #2 destination is a directory: create a new dir under that one
+      // #3 destination doesn't exist: create a new dir with that name
+      // #3 and #4 are only allowed if the dest path is not == or under src
+
+      if (destExists) {
+        if (!destIsDir) {
+          // #1 destination is a file: fail
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "returning false as src is a directory, but not dest");
+          }
+          return false;
+        } else {
+          // the destination dir exists
+          // destination for rename becomes a subdir of the target name
+          absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying src dir under dest dir to " + absoluteDst);
+          }
+        }
+      }
+      //the final destination directory is now know, so validate it for
+      //illegal moves
+
+      if (absoluteSrc.equals(absoluteDst)) {
+        //you can't rename a directory onto itself
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "Dest==source && isDir -failing");
+        }
+        return false;
+      }
+      if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
+        //you can't move a directory under itself
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "dst is equal to or under src dir -failing");
+        }
+        return false;
+      }
+    }
+    //here the dest path is set up -so rename
     return renameRecursive(absoluteSrc, absoluteDst);
   }
-  
+
   private boolean renameRecursive(Path src, Path dst) throws IOException {
     INode srcINode = store.retrieveINode(src);
     store.storeINode(dst, srcINode);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java

@@ -582,35 +582,58 @@ public class NativeS3FileSystem extends FileSystem {
   public boolean rename(Path src, Path dst) throws IOException {
 
     String srcKey = pathToKey(makeAbsolute(src));
+    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
 
     if (srcKey.length() == 0) {
       // Cannot rename root of file system
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(debugPreamble +
+                  "returning false as cannot rename the root of a filesystem");
+      }
       return false;
     }
 
-    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
-
+    //get status of source
+    boolean srcIsFile;
+    try {
+      srcIsFile = getFileStatus(src).isFile();
+    } catch (FileNotFoundException e) {
+      //bail out fast if the source does not exist
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(debugPreamble + "returning false as src does not exist");
+      }
+      return false;
+    }
     // Figure out the final destination
-    String dstKey;
+    String dstKey = pathToKey(makeAbsolute(dst));
+
     try {
       boolean dstIsFile = getFileStatus(dst).isFile();
       if (dstIsFile) {
+        //destination is a file.
+        //you can't copy a file or a directory onto an existing file
+        //except for the special case of dest==src, which is a no-op
         if(LOG.isDebugEnabled()) {
           LOG.debug(debugPreamble +
-              "returning false as dst is an already existing file");
+              "returning without rename as dst is an already existing file");
         }
-        return false;
+        //exit, returning true iff the rename is onto self
+        return srcKey.equals(dstKey);
       } else {
+        //destination exists and is a directory
         if(LOG.isDebugEnabled()) {
           LOG.debug(debugPreamble + "using dst as output directory");
         }
+        //destination goes under the dst path, with the name of the
+        //source entry
         dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
       }
     } catch (FileNotFoundException e) {
+      //destination does not exist => the source file or directory
+      //is copied over with the name of the destination
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble + "using dst as output destination");
       }
-      dstKey = pathToKey(makeAbsolute(dst));
       try {
         if (getFileStatus(dst.getParent()).isFile()) {
           if(LOG.isDebugEnabled()) {
@@ -628,16 +651,17 @@ public class NativeS3FileSystem extends FileSystem {
       }
     }
 
-    boolean srcIsFile;
-    try {
-      srcIsFile = getFileStatus(src).isFile();
-    } catch (FileNotFoundException e) {
-      if(LOG.isDebugEnabled()) {
-        LOG.debug(debugPreamble + "returning false as src does not exist");
+    //rename to self behavior follows Posix rules and is different
+    //for directories and files -the return code is driven by src type
+    if (srcKey.equals(dstKey)) {
+      //fully resolved destination key matches source: fail
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(debugPreamble + "renamingToSelf; returning true");
       }
-      return false;
+      return true;
     }
     if (srcIsFile) {
+      //source is a file; COPY then DELETE
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble +
             "src is file, so doing copy then delete in S3");
@@ -645,9 +669,19 @@ public class NativeS3FileSystem extends FileSystem {
       store.copy(srcKey, dstKey);
       store.delete(srcKey);
     } else {
+      //src is a directory
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble + "src is directory, so copying contents");
       }
+      //Verify dest is not a child of the parent
+      if (dstKey.startsWith(srcKey + "/")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(
+            debugPreamble + "cannot rename a directory to a subdirectory of self");
+        }
+        return false;
+      }
+      //create the subdir under the destination
       store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
 
       List<String> keysToDelete = new ArrayList<String>();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java

@@ -18,9 +18,10 @@
 
 package org.apache.hadoop.fs.shell;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -72,8 +73,12 @@ abstract class CommandWithDestination extends FsCommand {
    */
   protected void getLocalDestination(LinkedList<String> args)
   throws IOException {
-    String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
-    dst = new PathData(new File(pathString), getConf());
+    try {
+      String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
+      dst = new PathData(new URI(pathString), getConf());
+    } catch (URISyntaxException e) {
+      throw new IOException("unexpected URISyntaxException", e);
+    }
   }
 
   /**
@@ -218,7 +223,13 @@ abstract class CommandWithDestination extends FsCommand {
    */ 
   protected void copyFileToTarget(PathData src, PathData target) throws IOException {
     src.fs.setVerifyChecksum(verifyChecksum);
-    copyStreamToTarget(src.fs.open(src.path), target);
+    InputStream in = null;
+    try {
+      in = src.fs.open(src.path);
+      copyStreamToTarget(in, target);
+    } finally {
+      IOUtils.closeStream(in);
+    }
   }
   
   /**
@@ -295,4 +306,4 @@ abstract class CommandWithDestination extends FsCommand {
       processDeleteOnExit();
     }
   }
-}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -60,16 +62,20 @@ class CopyCommands {
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
-      cf.parse(args);
+      try {
+        CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
+        cf.parse(args);
 
-      delimiter = cf.getOpt("nl") ? "\n" : null;
+        delimiter = cf.getOpt("nl") ? "\n" : null;
 
-      dst = new PathData(new File(args.removeLast()), getConf());
-      if (dst.exists && dst.stat.isDirectory()) {
-        throw new PathIsDirectoryException(dst.toString());
+        dst = new PathData(new URI(args.removeLast()), getConf());
+        if (dst.exists && dst.stat.isDirectory()) {
+          throw new PathIsDirectoryException(dst.toString());
+        }
+        srcs = new LinkedList<PathData>();
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
       }
-      srcs = new LinkedList<PathData>();
     }
 
     @Override
@@ -188,9 +194,13 @@ class CopyCommands {
     // commands operating on local paths have no need for glob expansion
     @Override
     protected List<PathData> expandArgument(String arg) throws IOException {
-      List<PathData> items = new LinkedList<PathData>();
-      items.add(new PathData(new File(arg), getConf()));
-      return items;
+      try {
+        List<PathData> items = new LinkedList<PathData>();
+        items.add(new PathData(new URI(arg), getConf()));
+        return items;
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
+      }
     }
 
     @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -47,13 +48,14 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.JsonEncoding;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.util.MinimalPrettyPrinter;
 
 /**
- * Display contents of files 
+ * Display contents or checksums of files 
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -62,6 +64,7 @@ class Display extends FsCommand {
   public static void registerCommands(CommandFactory factory) {
     factory.addClass(Cat.class, "-cat");
     factory.addClass(Text.class, "-text");
+    factory.addClass(Checksum.class, "-checksum");
   }
 
   /**
@@ -161,6 +164,36 @@ class Display extends FsCommand {
       return i;
     }
   }
+  
+  public static class Checksum extends Display {
+    public static final String NAME = "checksum";
+    public static final String USAGE = "<src> ...";
+    public static final String DESCRIPTION =
+      "Dump checksum information for files that match the file\n" +
+      "pattern <src> to stdout. Note that this requires a round-trip\n" +
+      "to a datanode storing each block of the file, and thus is not\n" +
+      "efficient to run on a large number of files. The checksum of a\n" +
+      "file depends on its content, block size and the checksum\n" +
+      "algorithm and parameters used for creating the file.";
+
+    @Override
+    protected void processPath(PathData item) throws IOException {
+      if (item.stat.isDirectory()) {
+        throw new PathIsDirectoryException(item.toString());
+      }
+
+      FileChecksum checksum = item.fs.getFileChecksum(item.path);
+      if (checksum == null) {
+        out.printf("%s\tNONE\t\n", item.toString());
+      } else {
+        String checksumString = StringUtils.byteToHexString(
+            checksum.getBytes(), 0, checksum.getLength());
+        out.printf("%s\t%s\t%s\n",
+            item.toString(), checksum.getAlgorithmName(),
+            checksumString);
+      }
+    }
+  }
 
   protected class TextRecordInputStream extends InputStream {
     SequenceFile.Reader r;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java

@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Arrays;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathNotFoundException;
 
 /**
  * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
+ * PathData ensures that the returned path string will be the same as the
+ * one passed in during initialization (unlike Path objects which can
+ * modify the path string).
  * The stat field will be null if the path does not exist.
  */
 @InterfaceAudience.Private
@@ -51,6 +55,20 @@ public class PathData implements Comparable<PathData> {
   public FileStatus stat;
   public boolean exists;
 
+  /* True if the URI scheme was not present in the pathString but inferred.
+   */
+  private boolean inferredSchemeFromPath = false;
+
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern potentialUri =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern windowsNonUriAbsolutePath1 =
+      Pattern.compile("^/?[a-zA-Z]:\\\\");
+  private static final Pattern windowsNonUriAbsolutePath2 =
+      Pattern.compile("^/?[a-zA-Z]:/");
+
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
@@ -67,12 +85,12 @@ public class PathData implements Comparable<PathData> {
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
    * return exactly the same string used to initialize it
-   * @param localPath a local File
+   * @param localPath a local URI
    * @param conf the configuration file
    * @throws IOException if anything goes wrong...
    */
-  public PathData(File localPath, Configuration conf) throws IOException {
-    this(FileSystem.getLocal(conf), localPath.toString());
+  public PathData(URI localPath, Configuration conf) throws IOException {
+    this(FileSystem.getLocal(conf), localPath.getPath());
   }
 
   /**
@@ -86,6 +104,39 @@ public class PathData implements Comparable<PathData> {
     this(fs, pathString, lookupStat(fs, pathString, true));
   }
 
+  /**
+   * Validates the given Windows path.
+   * Throws IOException on failure.
+   * @param pathString a String of the path suppliued by the user.
+   */
+  private void ValidateWindowsPath(String pathString)
+  throws IOException
+  {
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Does it look like a URI? If so then just leave it alone.
+    if (potentialUri.matcher(pathString).find()) {
+      return;
+    }
+
+    // Looks like a relative path on Windows.
+    return;
+  }
+
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
@@ -100,6 +151,10 @@ public class PathData implements Comparable<PathData> {
     this.uri = stringToUri(pathString);
     this.path = fs.makeQualified(new Path(uri));
     setStat(stat);
+
+    if (Path.WINDOWS) {
+      ValidateWindowsPath(pathString);
+    }
   }
 
   // need a static method for the ctor above
@@ -236,7 +291,7 @@ public class PathData implements Comparable<PathData> {
    * Given a child of this directory, use the directory's path and the child's
    * basename to construct the string to the child.  This preserves relative
    * paths since Path will fully qualify.
-   * @param child a path contained within this directory
+   * @param childPath a path contained within this directory
    * @return String of the path relative to this directory
    */
   private String getStringForChildPath(Path childPath) {
@@ -283,7 +338,8 @@ public class PathData implements Comparable<PathData> {
       URI globUri = globPath.toUri();
       if (globUri.getScheme() != null) {
         globType = PathType.HAS_SCHEME;
-      } else if (new File(globUri.getPath()).isAbsolute()) {
+      } else if (!globUri.getPath().isEmpty() &&
+                 new Path(globUri.getPath()).isAbsolute()) {
         globType = PathType.SCHEMELESS_ABSOLUTE;
       } else {
         globType = PathType.RELATIVE;
@@ -386,7 +442,14 @@ public class PathData implements Comparable<PathData> {
     // No interpretation of symbols. Just decode % escaped chars.
     String decodedRemainder = uri.getSchemeSpecificPart();
 
-    if (scheme == null) {
+    // Drop the scheme if it was inferred to ensure fidelity between
+    // the input and output path strings.
+    if ((scheme == null) || (inferredSchemeFromPath)) {
+      if (Path.isWindowsAbsolutePath(decodedRemainder, true)) {
+        // Strip the leading '/' added in stringToUri so users see a valid
+        // Windows path.
+        decodedRemainder = decodedRemainder.substring(1);
+      }
       return decodedRemainder;
     } else {
       StringBuilder buffer = new StringBuilder();
@@ -409,13 +472,56 @@ public class PathData implements Comparable<PathData> {
     return ((LocalFileSystem)fs).pathToFile(path);
   }
 
+  /** Normalize the given Windows path string. This does the following:
+   *    1. Adds "file:" scheme for absolute paths.
+   *    2. Ensures the scheme-specific part starts with '/' per RFC2396.
+   *    3. Replaces backslash path separators with forward slashes.
+   *    @param pathString Path string supplied by the user.
+   *    @return normalized absolute path string. Returns the input string
+   *            if it is not a Windows absolute path.
+   */
+  private static String normalizeWindowsPath(String pathString)
+  throws IOException
+  {
+    if (!Path.WINDOWS) {
+      return pathString;
+    }
+
+    boolean slashed =
+        ((pathString.length() >= 1) && (pathString.charAt(0) == '/'));
+
+    // Is it a backslash-separated absolute path?
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      pathString = pathString.replace('\\', '/');
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a backslash-separated relative file path (no scheme and
+    // no drive-letter specifier)?
+    if ((pathString.indexOf(':') == -1) && (pathString.indexOf('\\') != -1)) {
+      pathString = pathString.replace('\\', '/');
+    }
+
+    return pathString;
+  }
+
   /** Construct a URI from a String with unescaped special characters
-   *  that have non-standard sematics. e.g. /, ?, #. A custom parsing
-   *  is needed to prevent misbihaviors.
+   *  that have non-standard semantics. e.g. /, ?, #. A custom parsing
+   *  is needed to prevent misbehavior.
    *  @param pathString The input path in string form
    *  @return URI
    */
-  private static URI stringToUri(String pathString) {
+  private static URI stringToUri(String pathString) throws IOException {
     // We can't use 'new URI(String)' directly. Since it doesn't do quoting
     // internally, the internal parser may fail or break the string at wrong
     // places. Use of multi-argument ctors will quote those chars for us,
@@ -424,9 +530,10 @@ public class PathData implements Comparable<PathData> {
     // parse uri components
     String scheme = null;
     String authority = null;
-
     int start = 0;
 
+    pathString = normalizeWindowsPath(pathString);
+
     // parse uri scheme, if any
     int colon = pathString.indexOf(':');
     int slash = pathString.indexOf('/');
@@ -445,8 +552,7 @@ public class PathData implements Comparable<PathData> {
       authority = pathString.substring(start, authEnd);
       start = authEnd;
     }
-
-    // uri path is the rest of the string. ? or # are not interpreated,
+    // uri path is the rest of the string. ? or # are not interpreted,
     // but any occurrence of them will be quoted by the URI ctor.
     String path = pathString.substring(start, pathString.length());
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java

@@ -61,6 +61,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
 import org.mortbay.io.Buffer;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Handler;
@@ -305,6 +306,13 @@ public class HttpServer implements FilterContainer {
     ret.setAcceptQueueSize(128);
     ret.setResolveNames(false);
     ret.setUseDirectBuffers(false);
+    if(Shell.WINDOWS) {
+      // result of setting the SO_REUSEADDR flag is different on Windows
+      // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+      // without this 2 NN's can start on the same machine and listen on 
+      // the same port with indeterminate routing of incoming requests to them
+      ret.setReuseAddress(false);
+    }
     ret.setHeaderBufferSize(1024*64);
     return ret;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java

@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 
 /** 
  * A byte sequence that is usable as a key or value.
- * It is resizable and distinguishes between the size of the seqeunce and
+ * It is resizable and distinguishes between the size of the sequence and
  * the current capacity. The hash function is the front of the md5 of the 
  * buffer. The sort order is the same as memcmp.
  */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java

@@ -203,8 +203,8 @@ public class ReadaheadPool {
       // It's also possible that we'll end up requesting readahead on some
       // other FD, which may be wasted work, but won't cause a problem.
       try {
-        NativeIO.posixFadviseIfPossible(fd, off, len,
-            NativeIO.POSIX_FADV_WILLNEED);
+        NativeIO.POSIX.posixFadviseIfPossible(fd, off, len,
+            NativeIO.POSIX.POSIX_FADV_WILLNEED);
       } catch (IOException ioe) {
         if (canceled) {
           // no big deal - the reader canceled the request and closed

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java

@@ -22,6 +22,7 @@ import java.io.FileDescriptor;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,7 +31,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.nativeio.Errno;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIOException;
-import org.apache.hadoop.io.nativeio.NativeIO.Stat;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
@@ -120,7 +121,7 @@ public class SecureIOUtils {
     FileInputStream fis = new FileInputStream(f);
     boolean success = false;
     try {
-      Stat stat = NativeIO.getFstat(fis.getFD());
+      Stat stat = NativeIO.POSIX.getFstat(fis.getFD());
       checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
           expectedGroup);
       success = true;
@@ -166,35 +167,30 @@ public class SecureIOUtils {
     if (skipSecurity) {
       return insecureCreateForWrite(f, permissions);
     } else {
-      // Use the native wrapper around open(2)
-      try {
-        FileDescriptor fd = NativeIO.open(f.getAbsolutePath(),
-          NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL,
-          permissions);
-        return new FileOutputStream(fd);
-      } catch (NativeIOException nioe) {
-        if (nioe.getErrno() == Errno.EEXIST) {
-          throw new AlreadyExistsException(nioe);
-        }
-        throw nioe;
-      }
+      return NativeIO.getCreateForWriteFileOutputStream(f, permissions);
     }
   }
 
   private static void checkStat(File f, String owner, String group, 
       String expectedOwner, 
       String expectedGroup) throws IOException {
+    boolean success = true;
     if (expectedOwner != null &&
         !expectedOwner.equals(owner)) {
-      throw new IOException(
-        "Owner '" + owner + "' for path " + f + " did not match " +
-        "expected owner '" + expectedOwner + "'");
+      if (Path.WINDOWS) {
+        UserGroupInformation ugi =
+            UserGroupInformation.createRemoteUser(expectedOwner);
+        final String adminsGroupString = "Administrators";
+        success = owner.equals(adminsGroupString)
+            && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString);
+      } else {
+        success = false;
+      }
     }
-    if (expectedGroup != null &&
-        !expectedGroup.equals(group)) {
+    if (!success) {
       throw new IOException(
-        "Group '" + group + "' for path " + f + " did not match " +
-        "expected group '" + expectedGroup + "'");
+          "Owner '" + owner + "' for path " + f + " did not match " +
+              "expected owner '" + expectedOwner + "'");
     }
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java

@@ -128,7 +128,7 @@ public class Text extends BinaryComparable
   /**
    * Returns the Unicode Scalar Value (32-bit integer value)
    * for the character at <code>position</code>. Note that this
-   * method avoids using the converter or doing String instatiation
+   * method avoids using the converter or doing String instantiation
    * @return the Unicode scalar value at position or -1
    *          if the position is invalid or points to a
    *          trailing byte
@@ -527,7 +527,7 @@ public class Text extends BinaryComparable
     int length = 0;
     int state = LEAD_BYTE;
     while (count < start+len) {
-      int aByte = ((int) utf8[count] & 0xFF);
+      int aByte = utf8[count] & 0xFF;
 
       switch (state) {
       case LEAD_BYTE:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java

@@ -23,108 +23,156 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Seekable;
 import org.apache.hadoop.io.compress.bzip2.BZip2Constants;
-import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor;
-import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor;
 import org.apache.hadoop.io.compress.bzip2.CBZip2InputStream;
 import org.apache.hadoop.io.compress.bzip2.CBZip2OutputStream;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
 
 /**
- * This class provides CompressionOutputStream and CompressionInputStream for
- * compression and decompression. Currently we dont have an implementation of
- * the Compressor and Decompressor interfaces, so those methods of
- * CompressionCodec which have a Compressor or Decompressor type argument, throw
- * UnsupportedOperationException.
+ * This class provides output and input streams for bzip2 compression
+ * and decompression.  It uses the native bzip2 library on the system
+ * if possible, else it uses a pure-Java implementation of the bzip2
+ * algorithm.  The configuration parameter
+ * io.compression.codec.bzip2.library can be used to control this
+ * behavior.
+ *
+ * In the pure-Java mode, the Compressor and Decompressor interfaces
+ * are not implemented.  Therefore, in that mode, those methods of
+ * CompressionCodec which have a Compressor or Decompressor type
+ * argument, throw UnsupportedOperationException.
+ *
+ * Currently, support for splittability is available only in the
+ * pure-Java mode; therefore, if a SplitCompressionInputStream is
+ * requested, the pure-Java implementation is used, regardless of the
+ * setting of the configuration parameter mentioned above.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public class BZip2Codec implements SplittableCompressionCodec {
+public class BZip2Codec implements Configurable, SplittableCompressionCodec {
 
   private static final String HEADER = "BZ";
   private static final int HEADER_LEN = HEADER.length();
   private static final String SUB_HEADER = "h9";
   private static final int SUB_HEADER_LEN = SUB_HEADER.length();
 
+  private Configuration conf;
+  
   /**
-  * Creates a new instance of BZip2Codec
+   * Set the configuration to be used by this object.
+   *
+   * @param conf the configuration object.
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+  
+  /**
+   * Return the configuration used by this object.
+   *
+   * @return the configuration object used by this objec.
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+  
+  /**
+  * Creates a new instance of BZip2Codec.
   */
   public BZip2Codec() { }
 
   /**
-  * Creates CompressionOutputStream for BZip2
-  *
-  * @param out
-  *            The output Stream
-  * @return The BZip2 CompressionOutputStream
-  * @throws java.io.IOException
-  *             Throws IO exception
-  */
+   * Create a {@link CompressionOutputStream} that will write to the given
+   * {@link OutputStream}.
+   *
+   * @param out        the location for the final output stream
+   * @return a stream the user can write uncompressed data to, to have it 
+   *         compressed
+   * @throws IOException
+   */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
       throws IOException {
-    return new BZip2CompressionOutputStream(out);
+    return createOutputStream(out, createCompressor());
   }
 
   /**
-  * Creates a compressor using given OutputStream.
+   * Create a {@link CompressionOutputStream} that will write to the given
+   * {@link OutputStream} with the given {@link Compressor}.
    *
-  * @return CompressionOutputStream
-    @throws java.io.IOException
+   * @param out        the location for the final output stream
+   * @param compressor compressor to use
+   * @return a stream the user can write uncompressed data to, to have it 
+   *         compressed
+   * @throws IOException
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
       Compressor compressor) throws IOException {
-    return createOutputStream(out);
+    return Bzip2Factory.isNativeBzip2Loaded(conf) ?
+      new CompressorStream(out, compressor, 
+                           conf.getInt("io.file.buffer.size", 4*1024)) :
+      new BZip2CompressionOutputStream(out);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return BZip2DummyCompressor.class
-  */
+   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+   *
+   * @return the type of compressor needed by this codec.
+   */
   @Override
-  public Class<? extends org.apache.hadoop.io.compress.Compressor> getCompressorType() {
-    return BZip2DummyCompressor.class;
+  public Class<? extends Compressor> getCompressorType() {
+    return Bzip2Factory.getBzip2CompressorType(conf);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return Compressor
-  */
+   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+   *
+   * @return a new compressor for use by this codec
+   */
   @Override
   public Compressor createCompressor() {
-    return new BZip2DummyCompressor();
+    return Bzip2Factory.getBzip2Compressor(conf);
   }
 
   /**
-  * Creates CompressionInputStream to be used to read off uncompressed data.
-  *
-  * @param in
-  *            The InputStream
-  * @return Returns CompressionInputStream for BZip2
-  * @throws java.io.IOException
-  *             Throws IOException
-  */
+   * Create a {@link CompressionInputStream} that will read from the given
+   * input stream and return a stream for uncompressed data.
+   *
+   * @param in the stream to read compressed bytes from
+   * @return a stream to read uncompressed bytes from
+   * @throws IOException
+   */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
       throws IOException {
-    return new BZip2CompressionInputStream(in);
+    return createInputStream(in, createDecompressor());
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return CompressionInputStream
-  */
+   * Create a {@link CompressionInputStream} that will read from the given
+   * {@link InputStream} with the given {@link Decompressor}, and return a 
+   * stream for uncompressed data.
+   *
+   * @param in           the stream to read compressed bytes from
+   * @param decompressor decompressor to use
+   * @return a stream to read uncompressed bytes from
+   * @throws IOException
+   */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
       Decompressor decompressor) throws IOException {
-    return createInputStream(in);
+    return Bzip2Factory.isNativeBzip2Loaded(conf) ? 
+      new DecompressorStream(in, decompressor,
+                             conf.getInt("io.file.buffer.size", 4*1024)) :
+      new BZip2CompressionInputStream(in);
   }
 
   /**
@@ -139,7 +187,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
    *
    * @return CompressionInputStream for BZip2 aligned at block boundaries
    */
-  @Override
   public SplitCompressionInputStream createInputStream(InputStream seekableIn,
       Decompressor decompressor, long start, long end, READ_MODE readMode)
       throws IOException {
@@ -184,23 +231,23 @@ public class BZip2Codec implements SplittableCompressionCodec {
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return BZip2DummyDecompressor.class
-  */
+   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+   *
+   * @return the type of decompressor needed by this codec.
+   */
   @Override
-  public Class<? extends org.apache.hadoop.io.compress.Decompressor> getDecompressorType() {
-    return BZip2DummyDecompressor.class;
+  public Class<? extends Decompressor> getDecompressorType() {
+    return Bzip2Factory.getBzip2DecompressorType(conf);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return Decompressor
-  */
+   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+   *
+   * @return a new decompressor for use by this codec
+   */
   @Override
   public Decompressor createDecompressor() {
-    return new BZip2DummyDecompressor();
+    return Bzip2Factory.getBzip2Decompressor(conf);
   }
 
   /**
@@ -236,7 +283,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }
 
-    @Override
     public void finish() throws IOException {
       if (needsReset) {
         // In the case that nothing is written to this stream, we still need to
@@ -256,14 +302,12 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }    
     
-    @Override
     public void resetState() throws IOException {
       // Cannot write to out at this point because out might not be ready
       // yet, as in SequenceFile.Writer implementation.
       needsReset = true;
     }
 
-    @Override
     public void write(int b) throws IOException {
       if (needsReset) {
         internalReset();
@@ -271,7 +315,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       this.output.write(b);
     }
 
-    @Override
     public void write(byte[] b, int off, int len) throws IOException {
       if (needsReset) {
         internalReset();
@@ -279,7 +322,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       this.output.write(b, off, len);
     }
 
-    @Override
     public void close() throws IOException {
       if (needsReset) {
         // In the case that nothing is written to this stream, we still need to
@@ -397,7 +439,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
 
     }// end of method
 
-    @Override
     public void close() throws IOException {
       if (!needsReset) {
         input.close();
@@ -433,7 +474,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
     *
     */
 
-    @Override
     public int read(byte[] b, int off, int len) throws IOException {
       if (needsReset) {
         internalReset();
@@ -457,7 +497,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
 
     }
 
-    @Override
     public int read() throws IOException {
       byte b[] = new byte[1];
       int result = this.read(b, 0, 1);
@@ -472,7 +511,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }    
     
-    @Override
     public void resetState() throws IOException {
       // Cannot read from bufferedIn at this point because bufferedIn
       // might not be ready
@@ -480,7 +518,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       needsReset = true;
     }
 
-    @Override
     public long getPos() {
       return this.compressedStreamPosition;
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java

@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.Compressor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A {@link Compressor} based on the popular 
+ * bzip2 compression algorithm.
+ * http://www.bzip2.org/
+ * 
+ */
+public class Bzip2Compressor implements Compressor {
+  private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024;
+
+  // The default values for the block size and work factor are the same 
+  // those in Julian Seward's original bzip2 implementation.
+  static final int DEFAULT_BLOCK_SIZE = 9;
+  static final int DEFAULT_WORK_FACTOR = 30;
+
+  private static final Log LOG = LogFactory.getLog(Bzip2Compressor.class);
+
+  // HACK - Use this as a global lock in the JNI layer.
+  private static Class<Bzip2Compressor> clazz = Bzip2Compressor.class;
+
+  private long stream;
+  private int blockSize;
+  private int workFactor;
+  private int directBufferSize;
+  private byte[] userBuf = null;
+  private int userBufOff = 0, userBufLen = 0;
+  private Buffer uncompressedDirectBuf = null;
+  private int uncompressedDirectBufOff = 0, uncompressedDirectBufLen = 0;
+  private boolean keepUncompressedBuf = false;
+  private Buffer compressedDirectBuf = null;
+  private boolean finish, finished;
+
+  /**
+   * Creates a new compressor with a default values for the
+   * compression block size and work factor.  Compressed data will be
+   * generated in bzip2 format.
+   */
+  public Bzip2Compressor() {
+    this(DEFAULT_BLOCK_SIZE, DEFAULT_WORK_FACTOR, DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  /**
+   * Creates a new compressor, taking settings from the configuration.
+   */
+  public Bzip2Compressor(Configuration conf) {
+    this(Bzip2Factory.getBlockSize(conf),
+         Bzip2Factory.getWorkFactor(conf),
+         DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  /** 
+   * Creates a new compressor using the specified block size.
+   * Compressed data will be generated in bzip2 format.
+   * 
+   * @param blockSize The block size to be used for compression.  This is
+   *        an integer from 1 through 9, which is multiplied by 100,000 to 
+   *        obtain the actual block size in bytes.
+   * @param workFactor This parameter is a threshold that determines when a 
+   *        fallback algorithm is used for pathological data.  It ranges from
+   *        0 to 250.
+   * @param directBufferSize Size of the direct buffer to be used.
+   */
+  public Bzip2Compressor(int blockSize, int workFactor, 
+                         int directBufferSize) {
+    this.blockSize = blockSize;
+    this.workFactor = workFactor;
+    this.directBufferSize = directBufferSize;
+    stream = init(blockSize, workFactor);
+    uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+  }
+
+  /**
+   * Prepare the compressor to be used in a new stream with settings defined in
+   * the given Configuration. It will reset the compressor's block size and
+   * and work factor.
+   * 
+   * @param conf Configuration storing new settings
+   */
+  @Override
+  public synchronized void reinit(Configuration conf) {
+    reset();
+    end(stream);
+    if (conf == null) {
+      stream = init(blockSize, workFactor);
+      return;
+    }
+    blockSize = Bzip2Factory.getBlockSize(conf);
+    workFactor = Bzip2Factory.getWorkFactor(conf);
+    stream = init(blockSize, workFactor);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Reinit compressor with new compression configuration");
+    }
+  }
+
+  @Override
+  public synchronized void setInput(byte[] b, int off, int len) {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    this.userBuf = b;
+    this.userBufOff = off;
+    this.userBufLen = len;
+    uncompressedDirectBufOff = 0;
+    setInputFromSavedData();
+    
+    // Reinitialize bzip2's output direct buffer.
+    compressedDirectBuf.limit(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+  }
+  
+  // Copy enough data from userBuf to uncompressedDirectBuf.
+  synchronized void setInputFromSavedData() {
+    int len = Math.min(userBufLen, uncompressedDirectBuf.remaining());
+    ((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff, len);
+    userBufLen -= len;
+    userBufOff += len;
+    uncompressedDirectBufLen = uncompressedDirectBuf.position();
+  }
+
+  @Override
+  public synchronized void setDictionary(byte[] b, int off, int len) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public synchronized boolean needsInput() {
+    // Compressed data still available?
+    if (compressedDirectBuf.remaining() > 0) {
+      return false;
+    }
+
+    // Uncompressed data available in either the direct buffer or user buffer?
+    if (keepUncompressedBuf && uncompressedDirectBufLen > 0)
+      return false;
+    
+    if (uncompressedDirectBuf.remaining() > 0) {
+      // Check if we have consumed all data in the user buffer.
+      if (userBufLen <= 0) {
+        return true;
+      } else {
+        // Copy enough data from userBuf to uncompressedDirectBuf.
+        setInputFromSavedData();
+        return uncompressedDirectBuf.remaining() > 0;
+      }
+    }
+    
+    return false;
+  }
+  
+  @Override
+  public synchronized void finish() {
+    finish = true;
+  }
+  
+  @Override
+  public synchronized boolean finished() {
+    // Check if bzip2 says it has finished and
+    // all compressed data has been consumed.
+    return (finished && compressedDirectBuf.remaining() == 0);
+  }
+
+  @Override
+  public synchronized int compress(byte[] b, int off, int len) 
+    throws IOException {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    // Check if there is compressed data.
+    int n = compressedDirectBuf.remaining();
+    if (n > 0) {
+      n = Math.min(n, len);
+      ((ByteBuffer)compressedDirectBuf).get(b, off, n);
+      return n;
+    }
+
+    // Re-initialize bzip2's output direct buffer.
+    compressedDirectBuf.rewind();
+    compressedDirectBuf.limit(directBufferSize);
+
+    // Compress the data.
+    n = deflateBytesDirect();
+    compressedDirectBuf.limit(n);
+    
+    // Check if bzip2 has consumed the entire input buffer.
+    // Set keepUncompressedBuf properly.
+    if (uncompressedDirectBufLen <= 0) { // bzip2 consumed all input
+      keepUncompressedBuf = false;
+      uncompressedDirectBuf.clear();
+      uncompressedDirectBufOff = 0;
+      uncompressedDirectBufLen = 0;
+    } else {
+      keepUncompressedBuf = true;
+    }
+
+    // Get at most 'len' bytes.
+    n = Math.min(n, len);
+    ((ByteBuffer)compressedDirectBuf).get(b, off, n);
+
+    return n;
+  }
+
+  /**
+   * Returns the total number of compressed bytes output so far.
+   *
+   * @return the total (non-negative) number of compressed bytes output so far
+   */
+  @Override
+  public synchronized long getBytesWritten() {
+    checkStream();
+    return getBytesWritten(stream);
+  }
+
+  /**
+   * Returns the total number of uncompressed bytes input so far.</p>
+   *
+   * @return the total (non-negative) number of uncompressed bytes input so far
+   */
+  @Override
+  public synchronized long getBytesRead() {
+    checkStream();
+    return getBytesRead(stream);
+  }
+
+  @Override
+  public synchronized void reset() {
+    checkStream();
+    end(stream);
+    stream = init(blockSize, workFactor);
+    finish = false;
+    finished = false;
+    uncompressedDirectBuf.rewind();
+    uncompressedDirectBufOff = uncompressedDirectBufLen = 0;
+    keepUncompressedBuf = false;
+    compressedDirectBuf.limit(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+    userBufOff = userBufLen = 0;
+  }
+  
+  @Override
+  public synchronized void end() {
+    if (stream != 0) {
+      end(stream);
+      stream = 0;
+    }
+  }
+  
+  static void initSymbols(String libname) {
+    initIDs(libname);
+  }
+
+  private void checkStream() {
+    if (stream == 0)
+      throw new NullPointerException();
+  }
+  
+  private native static void initIDs(String libname);
+  private native static long init(int blockSize, int workFactor);
+  private native int deflateBytesDirect();
+  private native static long getBytesRead(long strm);
+  private native static long getBytesWritten(long strm);
+  private native static void end(long strm);
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java

@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.io.compress.Decompressor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A {@link Decompressor} based on the popular 
+ * bzip2 compression algorithm.
+ * http://www.bzip2.org/
+ * 
+ */
+public class Bzip2Decompressor implements Decompressor {
+  private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024;
+  
+  private static final Log LOG = LogFactory.getLog(Bzip2Decompressor.class);
+
+  // HACK - Use this as a global lock in the JNI layer.
+  private static Class<Bzip2Decompressor> clazz = Bzip2Decompressor.class;
+  
+  private long stream;
+  private boolean conserveMemory;
+  private int directBufferSize;
+  private Buffer compressedDirectBuf = null;
+  private int compressedDirectBufOff, compressedDirectBufLen;
+  private Buffer uncompressedDirectBuf = null;
+  private byte[] userBuf = null;
+  private int userBufOff = 0, userBufLen = 0;
+  private boolean finished;
+
+  /**
+   * Creates a new decompressor.
+   */
+  public Bzip2Decompressor(boolean conserveMemory, int directBufferSize) {
+    this.conserveMemory = conserveMemory;
+    this.directBufferSize = directBufferSize;
+    compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+    
+    stream = init(conserveMemory ? 1 : 0);
+  }
+  
+  public Bzip2Decompressor() {
+    this(false, DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  @Override
+  public synchronized void setInput(byte[] b, int off, int len) {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+  
+    this.userBuf = b;
+    this.userBufOff = off;
+    this.userBufLen = len;
+    
+    setInputFromSavedData();
+    
+    // Reinitialize bzip2's output direct buffer.
+    uncompressedDirectBuf.limit(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+  }
+  
+  synchronized void setInputFromSavedData() {
+    compressedDirectBufOff = 0;
+    compressedDirectBufLen = userBufLen;
+    if (compressedDirectBufLen > directBufferSize) {
+      compressedDirectBufLen = directBufferSize;
+    }
+
+    // Reinitialize bzip2's input direct buffer.
+    compressedDirectBuf.rewind();
+    ((ByteBuffer)compressedDirectBuf).put(userBuf, userBufOff, 
+                                          compressedDirectBufLen);
+    
+    // Note how much data is being fed to bzip2.
+    userBufOff += compressedDirectBufLen;
+    userBufLen -= compressedDirectBufLen;
+  }
+
+  @Override
+  public synchronized void setDictionary(byte[] b, int off, int len) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public synchronized boolean needsInput() {
+    // Consume remaining compressed data?
+    if (uncompressedDirectBuf.remaining() > 0) {
+      return false;
+    }
+    
+    // Check if bzip2 has consumed all input.
+    if (compressedDirectBufLen <= 0) {
+      // Check if we have consumed all user-input.
+      if (userBufLen <= 0) {
+        return true;
+      } else {
+        setInputFromSavedData();
+      }
+    }
+    
+    return false;
+  }
+
+  @Override
+  public synchronized boolean needsDictionary() {
+    return false;
+  }
+
+  @Override
+  public synchronized boolean finished() {
+    // Check if bzip2 says it has finished and
+    // all compressed data has been consumed.
+    return (finished && uncompressedDirectBuf.remaining() == 0);
+  }
+
+  @Override
+  public synchronized int decompress(byte[] b, int off, int len) 
+    throws IOException {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    // Check if there is uncompressed data.
+    int n = uncompressedDirectBuf.remaining();
+    if (n > 0) {
+      n = Math.min(n, len);
+      ((ByteBuffer)uncompressedDirectBuf).get(b, off, n);
+      return n;
+    }
+    
+    // Re-initialize bzip2's output direct buffer.
+    uncompressedDirectBuf.rewind();
+    uncompressedDirectBuf.limit(directBufferSize);
+
+    // Decompress the data.
+    n = finished ? 0 : inflateBytesDirect();
+    uncompressedDirectBuf.limit(n);
+
+    // Get at most 'len' bytes.
+    n = Math.min(n, len);
+    ((ByteBuffer)uncompressedDirectBuf).get(b, off, n);
+
+    return n;
+  }
+  
+  /**
+   * Returns the total number of uncompressed bytes output so far.
+   *
+   * @return the total (non-negative) number of uncompressed bytes output so far
+   */
+  public synchronized long getBytesWritten() {
+    checkStream();
+    return getBytesWritten(stream);
+  }
+
+  /**
+   * Returns the total number of compressed bytes input so far.</p>
+   *
+   * @return the total (non-negative) number of compressed bytes input so far
+   */
+  public synchronized long getBytesRead() {
+    checkStream();
+    return getBytesRead(stream);
+  }
+
+  /**
+   * Returns the number of bytes remaining in the input buffers; normally
+   * called when finished() is true to determine amount of post-gzip-stream
+   * data.</p>
+   *
+   * @return the total (non-negative) number of unprocessed bytes in input
+   */
+  @Override
+  public synchronized int getRemaining() {
+    checkStream();
+    return userBufLen + getRemaining(stream);  // userBuf + compressedDirectBuf
+  }
+
+  /**
+   * Resets everything including the input buffers (user and direct).</p>
+   */
+  @Override
+  public synchronized void reset() {
+    checkStream();
+    end(stream);
+    stream = init(conserveMemory ? 1 : 0);
+    finished = false;
+    compressedDirectBufOff = compressedDirectBufLen = 0;
+    uncompressedDirectBuf.limit(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+    userBufOff = userBufLen = 0;
+  }
+
+  @Override
+  public synchronized void end() {
+    if (stream != 0) {
+      end(stream);
+      stream = 0;
+    }
+  }
+
+  static void initSymbols(String libname) {
+    initIDs(libname);
+  }
+
+  private void checkStream() {
+    if (stream == 0)
+      throw new NullPointerException();
+  }
+  
+  private native static void initIDs(String libname);
+  private native static long init(int conserveMemory);
+  private native int inflateBytesDirect();
+  private native static long getBytesRead(long strm);
+  private native static long getBytesWritten(long strm);
+  private native static int getRemaining(long strm);
+  private native static void end(long strm);
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Compressor;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor;
+import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor;
+import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor;
+
+/**
+ * A collection of factories to create the right 
+ * bzip2 compressor/decompressor instances.
+ * 
+ */
+public class Bzip2Factory {
+  private static final Log LOG = LogFactory.getLog(Bzip2Factory.class);
+
+  private static String bzip2LibraryName = "";
+  private static boolean nativeBzip2Loaded;
+  
+  /**
+   * Check if native-bzip2 code is loaded & initialized correctly and 
+   * can be loaded for this job.
+   * 
+   * @param conf configuration
+   * @return <code>true</code> if native-bzip2 is loaded & initialized 
+   *         and can be loaded for this job, else <code>false</code>
+   */
+  public static boolean isNativeBzip2Loaded(Configuration conf) {
+    String libname = conf.get("io.compression.codec.bzip2.library", 
+                              "system-native");
+    if (!bzip2LibraryName.equals(libname)) {
+      nativeBzip2Loaded = false;
+      bzip2LibraryName = libname;
+      if (libname.equals("java-builtin")) {
+        LOG.info("Using pure-Java version of bzip2 library");
+      } else if (conf.getBoolean(
+                CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, 
+                CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
+          NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          // Initialize the native library.
+          Bzip2Compressor.initSymbols(libname);
+          Bzip2Decompressor.initSymbols(libname);
+          nativeBzip2Loaded = true;
+          LOG.info("Successfully loaded & initialized native-bzip2 library " +
+                   libname);
+        } catch (Throwable t) {
+          LOG.warn("Failed to load/initialize native-bzip2 library " + 
+                   libname + ", will use pure-Java version");
+        }
+      }
+    }
+    return nativeBzip2Loaded;
+  }
+
+  /**
+   * Return the appropriate type of the bzip2 compressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate type of the bzip2 compressor.
+   */
+  public static Class<? extends Compressor> 
+  getBzip2CompressorType(Configuration conf) {
+    return isNativeBzip2Loaded(conf) ? 
+      Bzip2Compressor.class : BZip2DummyCompressor.class;
+  }
+  
+  /**
+   * Return the appropriate implementation of the bzip2 compressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate implementation of the bzip2 compressor.
+   */
+  public static Compressor getBzip2Compressor(Configuration conf) {
+    return isNativeBzip2Loaded(conf)? 
+      new Bzip2Compressor(conf) : new BZip2DummyCompressor();
+  }
+
+  /**
+   * Return the appropriate type of the bzip2 decompressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate type of the bzip2 decompressor.
+   */
+  public static Class<? extends Decompressor> 
+  getBzip2DecompressorType(Configuration conf) {
+    return  isNativeBzip2Loaded(conf) ? 
+      Bzip2Decompressor.class : BZip2DummyDecompressor.class;
+  }
+  
+  /**
+   * Return the appropriate implementation of the bzip2 decompressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate implementation of the bzip2 decompressor.
+   */
+  public static Decompressor getBzip2Decompressor(Configuration conf) {
+    return isNativeBzip2Loaded(conf) ? 
+      new Bzip2Decompressor() : new BZip2DummyDecompressor();
+  }
+
+  public static void setBlockSize(Configuration conf, int blockSize) {
+    conf.setInt("bzip2.compress.blocksize", blockSize);
+  }
+
+  public static int getBlockSize(Configuration conf) {
+    return conf.getInt("bzip2.compress.blocksize", 
+                       Bzip2Compressor.DEFAULT_BLOCK_SIZE);
+  }
+
+  public static void setWorkFactor(Configuration conf, int workFactor) {
+    conf.setInt("bzip2.compress.workfactor", workFactor);
+  }
+
+  public static int getWorkFactor(Configuration conf) {
+    return conf.getInt("bzip2.compress.workfactor", 
+                       Bzip2Compressor.DEFAULT_WORK_FACTOR);
+  }
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java

@@ -19,7 +19,10 @@ package org.apache.hadoop.io.nativeio;
 
 import java.io.File;
 import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -27,10 +30,13 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
 import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.Shell;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+
 /**
  * JNI wrappers for various native IO-related calls not available in Java.
  * These functions should generally be used alongside a fallback to another
@@ -39,81 +45,341 @@ import org.apache.commons.logging.LogFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class NativeIO {
-  // Flags for open() call from bits/fcntl.h
-  public static final int O_RDONLY   =    00;
-  public static final int O_WRONLY   =    01;
-  public static final int O_RDWR     =    02;
-  public static final int O_CREAT    =  0100;
-  public static final int O_EXCL     =  0200;
-  public static final int O_NOCTTY   =  0400;
-  public static final int O_TRUNC    = 01000;
-  public static final int O_APPEND   = 02000;
-  public static final int O_NONBLOCK = 04000;
-  public static final int O_SYNC   =  010000;
-  public static final int O_ASYNC  =  020000;
-  public static final int O_FSYNC = O_SYNC;
-  public static final int O_NDELAY = O_NONBLOCK;
-
-  // Flags for posix_fadvise() from bits/fcntl.h
-  /* No further special treatment.  */
-  public static final int POSIX_FADV_NORMAL = 0; 
-  /* Expect random page references.  */
-  public static final int POSIX_FADV_RANDOM = 1; 
-  /* Expect sequential page references.  */
-  public static final int POSIX_FADV_SEQUENTIAL = 2; 
-  /* Will need these pages.  */
-  public static final int POSIX_FADV_WILLNEED = 3; 
-  /* Don't need these pages.  */
-  public static final int POSIX_FADV_DONTNEED = 4; 
-  /* Data will be accessed once.  */
-  public static final int POSIX_FADV_NOREUSE = 5; 
-
-
-  /* Wait upon writeout of all pages
-     in the range before performing the
-     write.  */
-  public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
-  /* Initiate writeout of all those
-     dirty pages in the range which are
-     not presently under writeback.  */
-  public static final int SYNC_FILE_RANGE_WRITE = 2;
-
-  /* Wait upon writeout of all pages in
-     the range after performing the
-     write.  */
-  public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
+  public static class POSIX {
+    // Flags for open() call from bits/fcntl.h
+    public static final int O_RDONLY   =    00;
+    public static final int O_WRONLY   =    01;
+    public static final int O_RDWR     =    02;
+    public static final int O_CREAT    =  0100;
+    public static final int O_EXCL     =  0200;
+    public static final int O_NOCTTY   =  0400;
+    public static final int O_TRUNC    = 01000;
+    public static final int O_APPEND   = 02000;
+    public static final int O_NONBLOCK = 04000;
+    public static final int O_SYNC   =  010000;
+    public static final int O_ASYNC  =  020000;
+    public static final int O_FSYNC = O_SYNC;
+    public static final int O_NDELAY = O_NONBLOCK;
+
+    // Flags for posix_fadvise() from bits/fcntl.h
+    /* No further special treatment.  */
+    public static final int POSIX_FADV_NORMAL = 0;
+    /* Expect random page references.  */
+    public static final int POSIX_FADV_RANDOM = 1;
+    /* Expect sequential page references.  */
+    public static final int POSIX_FADV_SEQUENTIAL = 2;
+    /* Will need these pages.  */
+    public static final int POSIX_FADV_WILLNEED = 3;
+    /* Don't need these pages.  */
+    public static final int POSIX_FADV_DONTNEED = 4;
+    /* Data will be accessed once.  */
+    public static final int POSIX_FADV_NOREUSE = 5;
+
+
+    /* Wait upon writeout of all pages
+       in the range before performing the
+       write.  */
+    public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
+    /* Initiate writeout of all those
+       dirty pages in the range which are
+       not presently under writeback.  */
+    public static final int SYNC_FILE_RANGE_WRITE = 2;
+
+    /* Wait upon writeout of all pages in
+       the range after performing the
+       write.  */
+    public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
+
+    private static final Log LOG = LogFactory.getLog(NativeIO.class);
+
+    private static boolean nativeLoaded = false;
+    private static boolean fadvisePossible = true;
+    private static boolean syncFileRangePossible = true;
+
+    static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
+      "hadoop.workaround.non.threadsafe.getpwuid";
+    static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false;
+
+    private static long cacheTimeout = -1;
+
+    static {
+      if (NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          Configuration conf = new Configuration();
+          workaroundNonThreadSafePasswdCalls = conf.getBoolean(
+            WORKAROUND_NON_THREADSAFE_CALLS_KEY,
+            WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);
+
+          initNative();
+          nativeLoaded = true;
+
+          cacheTimeout = conf.getLong(
+            CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,
+            CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) *
+            1000;
+          LOG.debug("Initialized cache for IDs to User/Group mapping with a " +
+            " cache timeout of " + cacheTimeout/1000 + " seconds.");
+
+        } catch (Throwable t) {
+          // This can happen if the user has an older version of libhadoop.so
+          // installed - in this case we can continue without native IO
+          // after warning
+          LOG.error("Unable to initialize NativeIO libraries", t);
+        }
+      }
+    }
 
-  private static final Log LOG = LogFactory.getLog(NativeIO.class);
+    /**
+     * Return true if the JNI-based native IO extensions are available.
+     */
+    public static boolean isAvailable() {
+      return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
+    }
+
+    /** Wrapper around open(2) */
+    public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
+    /** Wrapper around fstat(2) */
+    private static native Stat fstat(FileDescriptor fd) throws IOException;
+
+    /** Native chmod implementation. On UNIX, it is a wrapper around chmod(2) */
+    private static native void chmodImpl(String path, int mode) throws IOException;
+
+    public static void chmod(String path, int mode) throws IOException {
+      if (!Shell.WINDOWS) {
+        chmodImpl(path, mode);
+      } else {
+        try {
+          chmodImpl(path, mode);
+        } catch (NativeIOException nioe) {
+          if (nioe.getErrorCode() == 3) {
+            throw new NativeIOException("No such file or directory",
+                Errno.ENOENT);
+          } else {
+            LOG.warn(String.format("NativeIO.chmod error (%d): %s",
+                nioe.getErrorCode(), nioe.getMessage()));
+            throw new NativeIOException("Unknown error", Errno.UNKNOWN);
+          }
+        }
+      }
+    }
+
+    /** Wrapper around posix_fadvise(2) */
+    static native void posix_fadvise(
+      FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;
+
+    /** Wrapper around sync_file_range(2) */
+    static native void sync_file_range(
+      FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException;
+
+    /**
+     * Call posix_fadvise on the given file descriptor. See the manpage
+     * for this syscall for more information. On systems where this
+     * call is not available, does nothing.
+     *
+     * @throws NativeIOException if there is an error with the syscall
+     */
+    public static void posixFadviseIfPossible(
+        FileDescriptor fd, long offset, long len, int flags)
+        throws NativeIOException {
+      if (nativeLoaded && fadvisePossible) {
+        try {
+          posix_fadvise(fd, offset, len, flags);
+        } catch (UnsupportedOperationException uoe) {
+          fadvisePossible = false;
+        } catch (UnsatisfiedLinkError ule) {
+          fadvisePossible = false;
+        }
+      }
+    }
+
+    /**
+     * Call sync_file_range on the given file descriptor. See the manpage
+     * for this syscall for more information. On systems where this
+     * call is not available, does nothing.
+     *
+     * @throws NativeIOException if there is an error with the syscall
+     */
+    public static void syncFileRangeIfPossible(
+        FileDescriptor fd, long offset, long nbytes, int flags)
+        throws NativeIOException {
+      if (nativeLoaded && syncFileRangePossible) {
+        try {
+          sync_file_range(fd, offset, nbytes, flags);
+        } catch (UnsupportedOperationException uoe) {
+          syncFileRangePossible = false;
+        } catch (UnsatisfiedLinkError ule) {
+          syncFileRangePossible = false;
+        }
+      }
+    }
+
+    /** Linux only methods used for getOwner() implementation */
+    private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException;
+    private static native String getUserName(long uid) throws IOException;
+
+    /**
+     * Result type of the fstat call
+     */
+    public static class Stat {
+      private int ownerId, groupId;
+      private String owner, group;
+      private int mode;
+
+      // Mode constants
+      public static final int S_IFMT = 0170000;      /* type of file */
+      public static final int   S_IFIFO  = 0010000;  /* named pipe (fifo) */
+      public static final int   S_IFCHR  = 0020000;  /* character special */
+      public static final int   S_IFDIR  = 0040000;  /* directory */
+      public static final int   S_IFBLK  = 0060000;  /* block special */
+      public static final int   S_IFREG  = 0100000;  /* regular */
+      public static final int   S_IFLNK  = 0120000;  /* symbolic link */
+      public static final int   S_IFSOCK = 0140000;  /* socket */
+      public static final int   S_IFWHT  = 0160000;  /* whiteout */
+      public static final int S_ISUID = 0004000;  /* set user id on execution */
+      public static final int S_ISGID = 0002000;  /* set group id on execution */
+      public static final int S_ISVTX = 0001000;  /* save swapped text even after use */
+      public static final int S_IRUSR = 0000400;  /* read permission, owner */
+      public static final int S_IWUSR = 0000200;  /* write permission, owner */
+      public static final int S_IXUSR = 0000100;  /* execute/search permission, owner */
+
+      Stat(int ownerId, int groupId, int mode) {
+        this.ownerId = ownerId;
+        this.groupId = groupId;
+        this.mode = mode;
+      }
+
+      @Override
+      public String toString() {
+        return "Stat(owner='" + owner + "', group='" + group + "'" +
+          ", mode=" + mode + ")";
+      }
+
+      public String getOwner() {
+        return owner;
+      }
+      public String getGroup() {
+        return group;
+      }
+      public int getMode() {
+        return mode;
+      }
+    }
+
+    /**
+     * Returns the file stat for a file descriptor.
+     *
+     * @param fd file descriptor.
+     * @return the file descriptor file stat.
+     * @throws IOException thrown if there was an IO error while obtaining the file stat.
+     */
+    public static Stat getFstat(FileDescriptor fd) throws IOException {
+      Stat stat = fstat(fd);
+      stat.owner = getName(IdCache.USER, stat.ownerId);
+      stat.group = getName(IdCache.GROUP, stat.groupId);
+      return stat;
+    }
+
+    private static String getName(IdCache domain, int id) throws IOException {
+      Map<Integer, CachedName> idNameCache = (domain == IdCache.USER)
+        ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE;
+      String name;
+      CachedName cachedName = idNameCache.get(id);
+      long now = System.currentTimeMillis();
+      if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) {
+        name = cachedName.name;
+      } else {
+        name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id);
+        if (LOG.isDebugEnabled()) {
+          String type = (domain == IdCache.USER) ? "UserName" : "GroupName";
+          LOG.debug("Got " + type + " " + name + " for ID " + id +
+            " from the native implementation");
+        }
+        cachedName = new CachedName(name, now);
+        idNameCache.put(id, cachedName);
+      }
+      return name;
+    }
+
+    static native String getUserName(int uid) throws IOException;
+    static native String getGroupName(int uid) throws IOException;
+
+    private static class CachedName {
+      final long timestamp;
+      final String name;
+
+      public CachedName(String name, long timestamp) {
+        this.name = name;
+        this.timestamp = timestamp;
+      }
+    }
+
+    private static final Map<Integer, CachedName> USER_ID_NAME_CACHE =
+      new ConcurrentHashMap<Integer, CachedName>();
+
+    private static final Map<Integer, CachedName> GROUP_ID_NAME_CACHE =
+      new ConcurrentHashMap<Integer, CachedName>();
+
+    private enum IdCache { USER, GROUP }
+  }
 
-  private static boolean nativeLoaded = false;
   private static boolean workaroundNonThreadSafePasswdCalls = false;
-  private static boolean fadvisePossible = true;
-  private static boolean syncFileRangePossible = true;
 
-  static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
-    "hadoop.workaround.non.threadsafe.getpwuid";
-  static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false;
 
-  private static long cacheTimeout = -1;
+  public static class Windows {
+    // Flags for CreateFile() call on Windows
+    public static final long GENERIC_READ = 0x80000000L;
+    public static final long GENERIC_WRITE = 0x40000000L;
+
+    public static final long FILE_SHARE_READ = 0x00000001L;
+    public static final long FILE_SHARE_WRITE = 0x00000002L;
+    public static final long FILE_SHARE_DELETE = 0x00000004L;
+
+    public static final long CREATE_NEW = 1;
+    public static final long CREATE_ALWAYS = 2;
+    public static final long OPEN_EXISTING = 3;
+    public static final long OPEN_ALWAYS = 4;
+    public static final long TRUNCATE_EXISTING = 5;
+
+    public static final long FILE_BEGIN = 0;
+    public static final long FILE_CURRENT = 1;
+    public static final long FILE_END = 2;
+
+    /** Wrapper around CreateFile() on Windows */
+    public static native FileDescriptor createFile(String path,
+        long desiredAccess, long shareMode, long creationDisposition)
+        throws IOException;
+
+    /** Wrapper around SetFilePointer() on Windows */
+    public static native long setFilePointer(FileDescriptor fd,
+        long distanceToMove, long moveMethod) throws IOException;
+
+    /** Windows only methods used for getOwner() implementation */
+    private static native String getOwner(FileDescriptor fd) throws IOException;
+
+    static {
+      if (NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          initNative();
+          nativeLoaded = true;
+        } catch (Throwable t) {
+          // This can happen if the user has an older version of libhadoop.so
+          // installed - in this case we can continue without native IO
+          // after warning
+          LOG.error("Unable to initialize NativeIO libraries", t);
+        }
+      }
+    }
+  }
+
+  private static final Log LOG = LogFactory.getLog(NativeIO.class);
+
+  private static boolean nativeLoaded = false;
 
   static {
     if (NativeCodeLoader.isNativeCodeLoaded()) {
       try {
-        Configuration conf = new Configuration();
-        workaroundNonThreadSafePasswdCalls = conf.getBoolean(
-          WORKAROUND_NON_THREADSAFE_CALLS_KEY,
-          WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);
-
         initNative();
         nativeLoaded = true;
-
-        cacheTimeout = conf.getLong(
-          CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,
-          CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) *
-          1000;
-        LOG.debug("Initialized cache for IDs to User/Group mapping with a" +
-          " cache timeout of " + cacheTimeout/1000 + " seconds.");
-
       } catch (Throwable t) {
         // This can happen if the user has an older version of libhadoop.so
         // installed - in this case we can continue without native IO
@@ -130,169 +396,161 @@ public class NativeIO {
     return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
   }
 
-  /** Wrapper around open(2) */
-  public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
-  /** Wrapper around fstat(2) */
-  private static native Stat fstat(FileDescriptor fd) throws IOException;
-  /** Wrapper around chmod(2) */
-  public static native void chmod(String path, int mode) throws IOException;
-
-  /** Wrapper around posix_fadvise(2) */
-  static native void posix_fadvise(
-    FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;
-
-  /** Wrapper around sync_file_range(2) */
-  static native void sync_file_range(
-    FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException;
-
   /** Initialize the JNI method ID and class ID cache */
   private static native void initNative();
 
-  /**
-   * Call posix_fadvise on the given file descriptor. See the manpage
-   * for this syscall for more information. On systems where this
-   * call is not available, does nothing.
-   *
-   * @throws NativeIOException if there is an error with the syscall
-   */
-  public static void posixFadviseIfPossible(
-      FileDescriptor fd, long offset, long len, int flags)
-      throws NativeIOException {
-    if (nativeLoaded && fadvisePossible) {
-      try {
-        posix_fadvise(fd, offset, len, flags);
-      } catch (UnsupportedOperationException uoe) {
-        fadvisePossible = false;
-      } catch (UnsatisfiedLinkError ule) {
-        fadvisePossible = false;
-      }
+  private static class CachedUid {
+    final long timestamp;
+    final String username;
+    public CachedUid(String username, long timestamp) {
+      this.timestamp = timestamp;
+      this.username = username;
     }
   }
-
-  /**
-   * Call sync_file_range on the given file descriptor. See the manpage
-   * for this syscall for more information. On systems where this
-   * call is not available, does nothing.
-   *
-   * @throws NativeIOException if there is an error with the syscall
-   */
-  public static void syncFileRangeIfPossible(
-      FileDescriptor fd, long offset, long nbytes, int flags)
-      throws NativeIOException {
-    if (nativeLoaded && syncFileRangePossible) {
-      try {
-        sync_file_range(fd, offset, nbytes, flags);
-      } catch (UnsupportedOperationException uoe) {
-        syncFileRangePossible = false;
-      } catch (UnsatisfiedLinkError ule) {
-        syncFileRangePossible = false;
+  private static final Map<Long, CachedUid> uidCache =
+      new ConcurrentHashMap<Long, CachedUid>();
+  private static long cacheTimeout;
+  private static boolean initialized = false;
+
+  public static String getOwner(FileDescriptor fd) throws IOException {
+    ensureInitialized();
+    if (Shell.WINDOWS) {
+      String owner = Windows.getOwner(fd);
+      int i = owner.indexOf('\\');
+      if (i != -1)
+        owner = owner.substring(i + 1);
+      return owner;
+    } else {
+      long uid = POSIX.getUIDforFDOwnerforOwner(fd);
+      CachedUid cUid = uidCache.get(uid);
+      long now = System.currentTimeMillis();
+      if (cUid != null && (cUid.timestamp + cacheTimeout) > now) {
+        return cUid.username;
       }
+      String user = POSIX.getUserName(uid);
+      LOG.info("Got UserName " + user + " for UID " + uid
+          + " from the native implementation");
+      cUid = new CachedUid(user, now);
+      uidCache.put(uid, cUid);
+      return user;
     }
   }
 
   /**
-   * Result type of the fstat call
+   * Create a FileInputStream that shares delete permission on the
+   * file opened, i.e. other process can delete the file the
+   * FileInputStream is reading. Only Windows implementation uses
+   * the native interface.
    */
-  public static class Stat {
-    private int ownerId, groupId;
-    private String owner, group;
-    private int mode;
-
-    // Mode constants
-    public static final int S_IFMT = 0170000;      /* type of file */
-    public static final int   S_IFIFO  = 0010000;  /* named pipe (fifo) */
-    public static final int   S_IFCHR  = 0020000;  /* character special */
-    public static final int   S_IFDIR  = 0040000;  /* directory */
-    public static final int   S_IFBLK  = 0060000;  /* block special */
-    public static final int   S_IFREG  = 0100000;  /* regular */
-    public static final int   S_IFLNK  = 0120000;  /* symbolic link */
-    public static final int   S_IFSOCK = 0140000;  /* socket */
-    public static final int   S_IFWHT  = 0160000;  /* whiteout */
-    public static final int S_ISUID = 0004000;  /* set user id on execution */
-    public static final int S_ISGID = 0002000;  /* set group id on execution */
-    public static final int S_ISVTX = 0001000;  /* save swapped text even after use */
-    public static final int S_IRUSR = 0000400;  /* read permission, owner */
-    public static final int S_IWUSR = 0000200;  /* write permission, owner */
-    public static final int S_IXUSR = 0000100;  /* execute/search permission, owner */
-
-    Stat(int ownerId, int groupId, int mode) {
-      this.ownerId = ownerId;
-      this.groupId = groupId;
-      this.mode = mode;
-    }
-
-    @Override
-    public String toString() {
-      return "Stat(owner='" + owner + "', group='" + group + "'" +
-        ", mode=" + mode + ")";
-    }
-
-    public String getOwner() {
-      return owner;
-    }
-    public String getGroup() {
-      return group;
-    }
-    public int getMode() {
-      return mode;
+  public static FileInputStream getShareDeleteFileInputStream(File f)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      // On Linux the default FileInputStream shares delete permission
+      // on the file opened.
+      //
+      return new FileInputStream(f);
+    } else {
+      // Use Windows native interface to create a FileInputStream that
+      // shares delete permission on the file opened.
+      //
+      FileDescriptor fd = Windows.createFile(
+          f.getAbsolutePath(),
+          Windows.GENERIC_READ,
+          Windows.FILE_SHARE_READ |
+              Windows.FILE_SHARE_WRITE |
+              Windows.FILE_SHARE_DELETE,
+          Windows.OPEN_EXISTING);
+      return new FileInputStream(fd);
     }
   }
 
-  static native String getUserName(int uid) throws IOException;
-
-  static native String getGroupName(int uid) throws IOException;
-
-  private static class CachedName {
-    final long timestamp;
-    final String name;
-
-    public CachedName(String name, long timestamp) {
-      this.name = name;
-      this.timestamp = timestamp;
+  /**
+   * Create a FileInputStream that shares delete permission on the
+   * file opened at a given offset, i.e. other process can delete
+   * the file the FileInputStream is reading. Only Windows implementation
+   * uses the native interface.
+   */
+  public static FileInputStream getShareDeleteFileInputStream(File f, long seekOffset)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      RandomAccessFile rf = new RandomAccessFile(f, "r");
+      if (seekOffset > 0) {
+        rf.seek(seekOffset);
+      }
+      return new FileInputStream(rf.getFD());
+    } else {
+      // Use Windows native interface to create a FileInputStream that
+      // shares delete permission on the file opened, and set it to the
+      // given offset.
+      //
+      FileDescriptor fd = NativeIO.Windows.createFile(
+          f.getAbsolutePath(),
+          NativeIO.Windows.GENERIC_READ,
+          NativeIO.Windows.FILE_SHARE_READ |
+              NativeIO.Windows.FILE_SHARE_WRITE |
+              NativeIO.Windows.FILE_SHARE_DELETE,
+          NativeIO.Windows.OPEN_EXISTING);
+      if (seekOffset > 0)
+        NativeIO.Windows.setFilePointer(fd, seekOffset, NativeIO.Windows.FILE_BEGIN);
+      return new FileInputStream(fd);
     }
   }
 
-  private static final Map<Integer, CachedName> USER_ID_NAME_CACHE =
-    new ConcurrentHashMap<Integer, CachedName>();
-
-  private static final Map<Integer, CachedName> GROUP_ID_NAME_CACHE =
-    new ConcurrentHashMap<Integer, CachedName>();
-
-  private enum IdCache { USER, GROUP }
-
-  private static String getName(IdCache domain, int id) throws IOException {
-    Map<Integer, CachedName> idNameCache = (domain == IdCache.USER)
-      ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE;
-    String name;
-    CachedName cachedName = idNameCache.get(id);
-    long now = System.currentTimeMillis();
-    if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) {
-      name = cachedName.name;
+  /**
+   * Create the specified File for write access, ensuring that it does not exist.
+   * @param f the file that we want to create
+   * @param permissions we want to have on the file (if security is enabled)
+   *
+   * @throws AlreadyExistsException if the file already exists
+   * @throws IOException if any other error occurred
+   */
+  public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      // Use the native wrapper around open(2)
+      try {
+        FileDescriptor fd = NativeIO.POSIX.open(f.getAbsolutePath(),
+            NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT
+                | NativeIO.POSIX.O_EXCL, permissions);
+        return new FileOutputStream(fd);
+      } catch (NativeIOException nioe) {
+        if (nioe.getErrno() == Errno.EEXIST) {
+          throw new AlreadyExistsException(nioe);
+        }
+        throw nioe;
+      }
     } else {
-      name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id);
-      if (LOG.isDebugEnabled()) {
-        String type = (domain == IdCache.USER) ? "UserName" : "GroupName";
-        LOG.debug("Got " + type + " " + name + " for ID " + id +
-          " from the native implementation");
+      // Use the Windows native APIs to create equivalent FileOutputStream
+      try {
+        FileDescriptor fd = NativeIO.Windows.createFile(f.getCanonicalPath(),
+            NativeIO.Windows.GENERIC_WRITE,
+            NativeIO.Windows.FILE_SHARE_DELETE
+                | NativeIO.Windows.FILE_SHARE_READ
+                | NativeIO.Windows.FILE_SHARE_WRITE,
+            NativeIO.Windows.CREATE_NEW);
+        NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions);
+        return new FileOutputStream(fd);
+      } catch (NativeIOException nioe) {
+        if (nioe.getErrorCode() == 80) {
+          // ERROR_FILE_EXISTS
+          // 80 (0x50)
+          // The file exists
+          throw new AlreadyExistsException(nioe);
+        }
+        throw nioe;
       }
-      cachedName = new CachedName(name, now);
-      idNameCache.put(id, cachedName);
     }
-    return name;
   }
 
-  /**
-   * Returns the file stat for a file descriptor.
-   *
-   * @param fd file descriptor.
-   * @return the file descriptor file stat.
-   * @throws IOException thrown if there was an IO error while obtaining the file stat.
-   */
-  public static Stat getFstat(FileDescriptor fd) throws IOException {
-    Stat stat = fstat(fd);
-    stat.owner = getName(IdCache.USER, stat.ownerId);
-    stat.group = getName(IdCache.GROUP, stat.groupId);
-    return stat;
+  private synchronized static void ensureInitialized() {
+    if (!initialized) {
+      cacheTimeout =
+          new Configuration().getLong("hadoop.security.uid.cache.secs",
+              4*60*60) * 1000;
+      LOG.info("Initialized cache for UID to User mapping with a cache" +
+          " timeout of " + cacheTimeout/1000 + " seconds.");
+      initialized = true;
+    }
   }
   
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java

@@ -18,20 +18,40 @@
 package org.apache.hadoop.io.nativeio;
 
 import java.io.IOException;
+import org.apache.hadoop.util.Shell;
+
 
 /**
  * An exception generated by a call to the native IO code.
  *
- * These exceptions simply wrap <i>errno</i> result codes.
+ * These exceptions simply wrap <i>errno</i> result codes on Linux,
+ * or the System Error Code on Windows.
  */
 public class NativeIOException extends IOException {
   private static final long serialVersionUID = 1L;
 
   private Errno errno;
 
+  // Java has no unsigned primitive error code. Use a signed 32-bit
+  // integer to hold the unsigned 32-bit integer.
+  private int errorCode;
+
   public NativeIOException(String msg, Errno errno) {
     super(msg);
     this.errno = errno;
+    // Windows error code is always set to ERROR_SUCCESS on Linux,
+    // i.e. no failure on Windows
+    this.errorCode = 0;
+  }
+
+  public NativeIOException(String msg, int errorCode) {
+    super(msg);
+    this.errorCode = errorCode;
+    this.errno = Errno.UNKNOWN;
+  }
+
+  public long getErrorCode() {
+    return errorCode;
   }
 
   public Errno getErrno() {
@@ -40,8 +60,10 @@ public class NativeIOException extends IOException {
 
   @Override
   public String toString() {
-    return errno.toString() + ": " + super.getMessage();
+    if (Shell.WINDOWS)
+      return errorCode + ": " + super.getMessage();
+    else
+      return errno.toString() + ": " + super.getMessage();
   }
 }
 
-

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -59,7 +59,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.io.retry.RetryPolicies;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
@@ -84,6 +83,7 @@ import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.CodedOutputStream;
 
 /** A client for an IPC service.  IPC calls take a single {@link Writable} as a
  * parameter, and return a {@link Writable} as their value.  A service runs on
@@ -243,7 +243,7 @@ public class Client {
       callComplete();
     }
     
-    public synchronized Writable getRpcResult() {
+    public synchronized Writable getRpcResponse() {
       return rpcResponse;
     }
   }
@@ -257,6 +257,7 @@ public class Client {
     private final ConnectionId remoteId;                // connection id
     private AuthMethod authMethod; // authentication method
     private Token<? extends TokenIdentifier> token;
+    private int serviceClass;
     private SaslRpcClient saslRpcClient;
     
     private Socket socket = null;                 // connected socket
@@ -279,7 +280,7 @@ public class Client {
     
     private final Object sendRpcRequestLock = new Object();
 
-    public Connection(ConnectionId remoteId) throws IOException {
+    public Connection(ConnectionId remoteId, int serviceClass) throws IOException {
       this.remoteId = remoteId;
       this.server = remoteId.getAddress();
       if (server.isUnresolved()) {
@@ -296,6 +297,7 @@ public class Client {
       this.tcpNoDelay = remoteId.getTcpNoDelay();
       this.doPing = remoteId.getDoPing();
       this.pingInterval = remoteId.getPingInterval();
+      this.serviceClass = serviceClass;
       if (LOG.isDebugEnabled()) {
         LOG.debug("The ping interval is " + this.pingInterval + " ms.");
       }
@@ -747,7 +749,9 @@ public class Client {
      * +----------------------------------+
      * |  "hrpc" 4 bytes                  |      
      * +----------------------------------+
-     * |  Version (1 bytes)               |      
+     * |  Version (1 byte)                |
+     * +----------------------------------+
+     * |  Service Class (1 byte)          |
      * +----------------------------------+
      * |  Authmethod (1 byte)             |      
      * +----------------------------------+
@@ -760,6 +764,7 @@ public class Client {
       // Write out the header, version and authentication method
       out.write(Server.HEADER.array());
       out.write(Server.CURRENT_VERSION);
+      out.write(serviceClass);
       authMethod.write(out);
       Server.IpcSerializationType.PROTOBUF.write(out);
       out.flush();
@@ -945,31 +950,58 @@ public class Client {
       touch();
       
       try {
-        RpcResponseHeaderProto response = 
+        int totalLen = in.readInt();
+        RpcResponseHeaderProto header = 
             RpcResponseHeaderProto.parseDelimitedFrom(in);
-        if (response == null) {
+        if (header == null) {
           throw new IOException("Response is null.");
         }
+        int headerLen = header.getSerializedSize();
+        headerLen += CodedOutputStream.computeRawVarint32Size(headerLen);
 
-        int callId = response.getCallId();
+        int callId = header.getCallId();
         if (LOG.isDebugEnabled())
           LOG.debug(getName() + " got value #" + callId);
 
         Call call = calls.get(callId);
-        RpcStatusProto status = response.getStatus();
+        RpcStatusProto status = header.getStatus();
         if (status == RpcStatusProto.SUCCESS) {
           Writable value = ReflectionUtils.newInstance(valueClass, conf);
           value.readFields(in);                 // read value
           call.setRpcResponse(value);
           calls.remove(callId);
-        } else if (status == RpcStatusProto.ERROR) {
-          call.setException(new RemoteException(WritableUtils.readString(in),
-                                                WritableUtils.readString(in)));
-          calls.remove(callId);
-        } else if (status == RpcStatusProto.FATAL) {
-          // Close the connection
-          markClosed(new RemoteException(WritableUtils.readString(in), 
-                                         WritableUtils.readString(in)));
+          
+          // verify that length was correct
+          // only for ProtobufEngine where len can be verified easily
+          if (call.getRpcResponse() instanceof ProtobufRpcEngine.RpcWrapper) {
+            ProtobufRpcEngine.RpcWrapper resWrapper = 
+                (ProtobufRpcEngine.RpcWrapper) call.getRpcResponse();
+            if (totalLen != headerLen + resWrapper.getLength()) { 
+              throw new RpcClientException(
+                  "RPC response length mismatch on rpc success");
+            }
+          }
+        } else { // Rpc Request failed
+          // Verify that length was correct
+          if (totalLen != headerLen) {
+            throw new RpcClientException(
+                "RPC response length mismatch on rpc error");
+          }
+          
+          final String exceptionClassName = header.hasExceptionClassName() ?
+                header.getExceptionClassName() : 
+                  "ServerDidNotSetExceptionClassName";
+          final String errorMsg = header.hasErrorMsg() ? 
+                header.getErrorMsg() : "ServerDidNotSetErrorMsg" ;
+          RemoteException re = 
+              new RemoteException(exceptionClassName, errorMsg);
+          if (status == RpcStatusProto.ERROR) {
+            call.setException(re);
+            calls.remove(callId);
+          } else if (status == RpcStatusProto.FATAL) {
+            // Close the connection
+            markClosed(re);
+          }
         }
       } catch (IOException e) {
         markClosed(e);
@@ -1152,19 +1184,33 @@ public class Client {
 
   
   /**
-   * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress, 
+   * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress,
    * Class, UserGroupInformation, int, Configuration)}
    * except that rpcKind is writable.
    */
-  public Writable call(Writable param, InetSocketAddress addr, 
+  public Writable call(Writable param, InetSocketAddress addr,
       Class<?> protocol, UserGroupInformation ticket,
-      int rpcTimeout, Configuration conf)  
+      int rpcTimeout, Configuration conf)
       throws InterruptedException, IOException {
-        ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
+    ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
         ticket, rpcTimeout, conf);
     return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
   }
   
+  /**
+   * Same as {@link #call(Writable, InetSocketAddress,
+   * Class, UserGroupInformation, int, Configuration)}
+   * except that specifying serviceClass.
+   */
+  public Writable call(Writable param, InetSocketAddress addr,
+      Class<?> protocol, UserGroupInformation ticket,
+      int rpcTimeout, int serviceClass, Configuration conf)
+      throws InterruptedException, IOException {
+    ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
+        ticket, rpcTimeout, conf);
+    return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, serviceClass);
+  }
+
   /**
    * Make a call, passing <code>param</code>, to the IPC server running at
    * <code>address</code> which is servicing the <code>protocol</code> protocol,
@@ -1191,6 +1237,22 @@ public class Client {
      return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
   }
   
+  /**
+   * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
+   * <code>remoteId</code>, returning the rpc respond.
+   *
+   * @param rpcKind
+   * @param rpcRequest -  contains serialized method and method parameters
+   * @param remoteId - the target rpc server
+   * @returns the rpc response
+   * Throws exceptions if there are network problems or if the remote code 
+   * threw an exception.
+   */
+  public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+      ConnectionId remoteId) throws InterruptedException, IOException {
+    return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT);
+  }
+
   /** 
    * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
    * <code>remoteId</code>, returning the rpc respond.
@@ -1198,14 +1260,16 @@ public class Client {
    * @param rpcKind
    * @param rpcRequest -  contains serialized method and method parameters
    * @param remoteId - the target rpc server
+   * @param serviceClass - service class for RPC
    * @returns the rpc response
    * Throws exceptions if there are network problems or if the remote code 
    * threw an exception.
    */
   public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
-      ConnectionId remoteId) throws InterruptedException, IOException {
+      ConnectionId remoteId, int serviceClass)
+      throws InterruptedException, IOException {
     Call call = new Call(rpcKind, rpcRequest);
-    Connection connection = getConnection(remoteId, call);
+    Connection connection = getConnection(remoteId, call, serviceClass);
     try {
       connection.sendRpcRequest(call);                 // send the rpc request
     } catch (RejectedExecutionException e) {
@@ -1245,7 +1309,7 @@ public class Client {
                   call.error);
         }
       } else {
-        return call.getRpcResult();
+        return call.getRpcResponse();
       }
     }
   }
@@ -1262,7 +1326,7 @@ public class Client {
   /** Get a connection from the pool, or create a new one and add it to the
    * pool.  Connections to a given ConnectionId are reused. */
   private Connection getConnection(ConnectionId remoteId,
-                                   Call call)
+                                   Call call, int serviceClass)
                                    throws IOException, InterruptedException {
     if (!running.get()) {
       // the client is stopped
@@ -1277,7 +1341,7 @@ public class Client {
       synchronized (connections) {
         connection = connections.get(remoteId);
         if (connection == null) {
-          connection = new Connection(remoteId);
+          connection = new Connection(remoteId, serviceClass);
           connections.put(remoteId, connection);
         }
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.ipc;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.net.InetSocketAddress;
@@ -39,7 +40,7 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.ipc.RPC.RpcInvoker;
-import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestProto;
+import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -47,7 +48,9 @@ import org.apache.hadoop.util.ProtoUtil;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.AbstractMessageLite;
 import com.google.protobuf.BlockingService;
+import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.Descriptors.MethodDescriptor;
 import com.google.protobuf.Message;
 import com.google.protobuf.ServiceException;
@@ -128,25 +131,12 @@ public class ProtobufRpcEngine implements RpcEngine {
           .getProtocolVersion(protocol);
     }
 
-    private RequestProto constructRpcRequest(Method method,
-        Object[] params) throws ServiceException {
-      RequestProto rpcRequest;
-      RequestProto.Builder builder = RequestProto
+    private RequestHeaderProto constructRpcRequestHeader(Method method) {
+      RequestHeaderProto.Builder builder = RequestHeaderProto
           .newBuilder();
       builder.setMethodName(method.getName());
+     
 
-      if (params.length != 2) { // RpcController + Message
-        throw new ServiceException("Too many parameters for request. Method: ["
-            + method.getName() + "]" + ", Expected: 2, Actual: "
-            + params.length);
-      }
-      if (params[1] == null) {
-        throw new ServiceException("null param while calling Method: ["
-            + method.getName() + "]");
-      }
-
-      Message param = (Message) params[1];
-      builder.setRequest(param.toByteString());
       // For protobuf, {@code protocol} used when creating client side proxy is
       // the interface extending BlockingInterface, which has the annotations 
       // such as ProtocolName etc.
@@ -160,8 +150,7 @@ public class ProtobufRpcEngine implements RpcEngine {
       // For PB this may limit the use of mixins on client side.
       builder.setDeclaringClassProtocolName(protocolName);
       builder.setClientProtocolVersion(clientProtocolVersion);
-      rpcRequest = builder.build();
-      return rpcRequest;
+      return builder.build();
     }
 
     /**
@@ -189,8 +178,18 @@ public class ProtobufRpcEngine implements RpcEngine {
       if (LOG.isDebugEnabled()) {
         startTime = Time.now();
       }
+      
+      if (args.length != 2) { // RpcController + Message
+        throw new ServiceException("Too many parameters for request. Method: ["
+            + method.getName() + "]" + ", Expected: 2, Actual: "
+            + args.length);
+      }
+      if (args[1] == null) {
+        throw new ServiceException("null param while calling Method: ["
+            + method.getName() + "]");
+      }
 
-      RequestProto rpcRequest = constructRpcRequest(method, args);
+      RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method);
       RpcResponseWrapper val = null;
       
       if (LOG.isTraceEnabled()) {
@@ -198,9 +197,12 @@ public class ProtobufRpcEngine implements RpcEngine {
             remoteId + ": " + method.getName() +
             " {" + TextFormat.shortDebugString((Message) args[1]) + "}");
       }
+
+
+      Message theRequest = (Message) args[1];
       try {
         val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
-            new RpcRequestWrapper(rpcRequest), remoteId);
+            new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId);
 
       } catch (Throwable e) {
         if (LOG.isTraceEnabled()) {
@@ -226,7 +228,7 @@ public class ProtobufRpcEngine implements RpcEngine {
       Message returnMessage;
       try {
         returnMessage = prototype.newBuilderForType()
-            .mergeFrom(val.responseMessage).build();
+            .mergeFrom(val.theResponseRead).build();
 
         if (LOG.isTraceEnabled()) {
           LOG.trace(Thread.currentThread().getId() + ": Response <- " +
@@ -267,6 +269,9 @@ public class ProtobufRpcEngine implements RpcEngine {
     }
   }
 
+  interface RpcWrapper extends Writable {
+    int getLength();
+  }
   /**
    * Wrapper for Protocol Buffer Requests
    * 
@@ -274,21 +279,26 @@ public class ProtobufRpcEngine implements RpcEngine {
    * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC} 
    * use type Writable as a wrapper to work across multiple RpcEngine kinds.
    */
-  private static class RpcRequestWrapper implements Writable {
-    RequestProto message;
+  private static class RpcRequestWrapper implements RpcWrapper {
+    RequestHeaderProto requestHeader;
+    Message theRequest; // for clientSide, the request is here
+    byte[] theRequestRead; // for server side, the request is here
 
     @SuppressWarnings("unused")
     public RpcRequestWrapper() {
     }
 
-    RpcRequestWrapper(RequestProto message) {
-      this.message = message;
+    RpcRequestWrapper(RequestHeaderProto requestHeader, Message theRequest) {
+      this.requestHeader = requestHeader;
+      this.theRequest = theRequest;
     }
 
     @Override
     public void write(DataOutput out) throws IOException {
-      ((Message)message).writeDelimitedTo(
-          DataOutputOutputStream.constructOutputStream(out));
+      OutputStream os = DataOutputOutputStream.constructOutputStream(out);
+      
+      ((Message)requestHeader).writeDelimitedTo(os);
+      theRequest.writeDelimitedTo(os);
     }
 
     @Override
@@ -296,13 +306,32 @@ public class ProtobufRpcEngine implements RpcEngine {
       int length = ProtoUtil.readRawVarint32(in);
       byte[] bytes = new byte[length];
       in.readFully(bytes);
-      message = RequestProto.parseFrom(bytes);
+      requestHeader = RequestHeaderProto.parseFrom(bytes);
+      length = ProtoUtil.readRawVarint32(in);
+      theRequestRead = new byte[length];
+      in.readFully(theRequestRead);
     }
     
     @Override
     public String toString() {
-      return message.getDeclaringClassProtocolName() + "." +
-          message.getMethodName();
+      return requestHeader.getDeclaringClassProtocolName() + "." +
+          requestHeader.getMethodName();
+    }
+
+    @Override
+    public int getLength() {
+      int headerLen = requestHeader.getSerializedSize();
+      int reqLen;
+      if (theRequest != null) {
+        reqLen = theRequest.getSerializedSize();
+      } else if (theRequestRead != null ) {
+        reqLen = theRequestRead.length;
+      } else {
+        throw new IllegalArgumentException(
+            "getLenght on uninilialized RpcWrapper");      
+      }
+      return CodedOutputStream.computeRawVarint32Size(headerLen) +  headerLen
+          + CodedOutputStream.computeRawVarint32Size(reqLen) + reqLen;
     }
   }
 
@@ -313,29 +342,43 @@ public class ProtobufRpcEngine implements RpcEngine {
    * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC} 
    * use type Writable as a wrapper to work across multiple RpcEngine kinds.
    */
-  private static class RpcResponseWrapper implements Writable {
-    byte[] responseMessage;
+  private static class RpcResponseWrapper implements RpcWrapper {
+    Message theResponse; // for senderSide, the response is here
+    byte[] theResponseRead; // for receiver side, the response is here
 
     @SuppressWarnings("unused")
     public RpcResponseWrapper() {
     }
 
     public RpcResponseWrapper(Message message) {
-      this.responseMessage = message.toByteArray();
+      this.theResponse = message;
     }
 
     @Override
     public void write(DataOutput out) throws IOException {
-      out.writeInt(responseMessage.length);
-      out.write(responseMessage);     
+      OutputStream os = DataOutputOutputStream.constructOutputStream(out);
+      theResponse.writeDelimitedTo(os);   
     }
 
     @Override
     public void readFields(DataInput in) throws IOException {
-      int length = in.readInt();
-      byte[] bytes = new byte[length];
-      in.readFully(bytes);
-      responseMessage = bytes;
+      int length = ProtoUtil.readRawVarint32(in);
+      theResponseRead = new byte[length];
+      in.readFully(theResponseRead);
+    }
+    
+    @Override
+    public int getLength() {
+      int resLen;
+      if (theResponse != null) {
+        resLen = theResponse.getSerializedSize();
+      } else if (theResponseRead != null ) {
+        resLen = theResponseRead.length;
+      } else {
+        throw new IllegalArgumentException(
+            "getLenght on uninilialized RpcWrapper");      
+      }
+      return CodedOutputStream.computeRawVarint32Size(resLen) + resLen;
     }
   }
 
@@ -434,7 +477,7 @@ public class ProtobufRpcEngine implements RpcEngine {
       public Writable call(RPC.Server server, String connectionProtocolName,
           Writable writableRequest, long receiveTime) throws Exception {
         RpcRequestWrapper request = (RpcRequestWrapper) writableRequest;
-        RequestProto rpcRequest = request.message;
+        RequestHeaderProto rpcRequest = request.requestHeader;
         String methodName = rpcRequest.getMethodName();
         
         
@@ -474,7 +517,8 @@ public class ProtobufRpcEngine implements RpcEngine {
         }
         Message prototype = service.getRequestPrototype(methodDescriptor);
         Message param = prototype.newBuilderForType()
-            .mergeFrom(rpcRequest.getRequest()).build();
+            .mergeFrom(request.theRequestRead).build();
+        
         Message result;
         try {
           long startTime = Time.now();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java

@@ -77,12 +77,12 @@ import com.google.protobuf.BlockingService;
 @InterfaceAudience.LimitedPrivate(value = { "Common", "HDFS", "MapReduce", "Yarn" })
 @InterfaceStability.Evolving
 public class RPC {
+  final static int RPC_SERVICE_CLASS_DEFAULT = 0;
   public enum RpcKind {
     RPC_BUILTIN ((short) 1),         // Used for built in calls by tests
     RPC_WRITABLE ((short) 2),        // Use WritableRpcEngine 
     RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
     final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
-    private static final short FIRST_INDEX = RPC_BUILTIN.value;    
     public final short value; //TODO make it private
 
     RpcKind(short val) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -24,6 +24,7 @@ import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
 import java.net.BindException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -72,6 +73,7 @@ import org.apache.hadoop.conf.Configuration.IntegerRanges;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
@@ -106,6 +108,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.CodedOutputStream;
 
 /** An abstract IPC service.  IPC calls take a single {@link Writable} as a
  * parameter, and return a {@link Writable} as their value.  A service runs on
@@ -201,7 +204,8 @@ public abstract class Server {
   // 6 : Made RPC Request header explicit
   // 7 : Changed Ipc Connection Header to use Protocol buffers
   // 8 : SASL server always sends a final response
-  public static final byte CURRENT_VERSION = 8;
+  // 9 : Changes to protocol for HADOOP-8990
+  public static final byte CURRENT_VERSION = 9;
 
   /**
    * Initial and max size of response buffer
@@ -313,6 +317,14 @@ public abstract class Server {
     return (addr == null) ? null : addr.getHostAddress();
   }
 
+  /** Returns the RPC remote user when invoked inside an RPC.  Note this
+   *  may be different than the current user if called within another doAs
+   *  @return connection's UGI or null if not an RPC
+   */
+  public static UserGroupInformation getRemoteUser() {
+    Call call = CurCall.get();
+    return (call != null) ? call.connection.user : null;
+  }
  
   /** Return true if the invocation was through an RPC.
    */
@@ -426,6 +438,11 @@ public abstract class Server {
     return Arrays.asList(handlers);
   }
 
+  @VisibleForTesting
+  List<Connection> getConnections() {
+    return connectionList;
+  }
+
   /**
    * Refresh the service authorization ACL for the service handled by this server.
    */
@@ -1092,6 +1109,7 @@ public abstract class Server {
     private ByteBuffer connectionHeaderBuf = null;
     private ByteBuffer unwrappedData;
     private ByteBuffer unwrappedDataLengthBuffer;
+    private int serviceClass;
     
     UserGroupInformation user = null;
     public UserGroupInformation attemptingUser = null; // user name before auth
@@ -1219,7 +1237,8 @@ public abstract class Server {
           rpcMetrics.incrAuthenticationFailures();
           String clientIP = this.toString();
           // attempting user could be null
-          AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser);
+          AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser +
+            " (" + e.getLocalizedMessage() + ")");
           throw e;
         }
         if (saslServer.isComplete() && replyToken == null) {
@@ -1302,14 +1321,17 @@ public abstract class Server {
         if (!connectionHeaderRead) {
           //Every connection is expected to send the header.
           if (connectionHeaderBuf == null) {
-            connectionHeaderBuf = ByteBuffer.allocate(3);
+            connectionHeaderBuf = ByteBuffer.allocate(4);
           }
           count = channelRead(channel, connectionHeaderBuf);
           if (count < 0 || connectionHeaderBuf.remaining() > 0) {
             return count;
           }
           int version = connectionHeaderBuf.get(0);
-          byte[] method = new byte[] {connectionHeaderBuf.get(1)};
+          // TODO we should add handler for service class later
+          this.setServiceClass(connectionHeaderBuf.get(1));
+
+          byte[] method = new byte[] {connectionHeaderBuf.get(2)};
           authMethod = AuthMethod.read(new DataInputStream(
               new ByteArrayInputStream(method)));
           dataLengthBuffer.flip();
@@ -1333,7 +1355,7 @@ public abstract class Server {
           }
           
           IpcSerializationType serializationType = IpcSerializationType
-              .fromByte(connectionHeaderBuf.get(2));
+              .fromByte(connectionHeaderBuf.get(3));
           if (serializationType != IpcSerializationType.PROTOBUF) {
             respondUnsupportedSerialization(serializationType);
             return -1;
@@ -1503,10 +1525,15 @@ public abstract class Server {
       " cannot communicate with client version " + clientVersion;
       ByteArrayOutputStream buffer = new ByteArrayOutputStream();
       
-      if (clientVersion >= 3) {
+      if (clientVersion >= 9) {
+        // Versions >>9  understand the normal response
         Call fakeCall =  new Call(-1, null, this);
-        // Versions 3 and greater can interpret this exception
-        // response in the same manner
+        setupResponse(buffer, fakeCall, RpcStatusProto.FATAL,
+            null, VersionMismatch.class.getName(), errMsg);
+        responder.doRespond(fakeCall);
+      } else if (clientVersion >= 3) {
+        Call fakeCall =  new Call(-1, null, this);
+        // Versions 3 to 8 use older response
         setupResponseOldVersionFatal(buffer, fakeCall,
             null, VersionMismatch.class.getName(), errMsg);
 
@@ -1553,9 +1580,6 @@ public abstract class Server {
       UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext);
       if (saslServer == null) {
         user = protocolUser;
-        if (user != null) {
-          user.setAuthenticationMethod(AuthMethod.SIMPLE);
-        }
       } else {
         // user is authenticated
         user.setAuthenticationMethod(authMethod);
@@ -1721,6 +1745,22 @@ public abstract class Server {
       return true;
     }
     
+    /**
+     * Get service class for connection
+     * @return the serviceClass
+     */
+    public int getServiceClass() {
+      return serviceClass;
+    }
+
+    /**
+     * Set service class for connection
+     * @param serviceClass the serviceClass to set
+     */
+    public void setServiceClass(int serviceClass) {
+      this.serviceClass = serviceClass;
+    }
+
     private synchronized void close() throws IOException {
       disposeSasl();
       data = null;
@@ -1783,6 +1823,9 @@ public abstract class Server {
                   );
             }
           } catch (Throwable e) {
+            if (e instanceof UndeclaredThrowableException) {
+              e = e.getCause();
+            }
             String logMsg = getName() + ", call " + call + ": error: " + e;
             if (e instanceof RuntimeException || e instanceof Error) {
               // These exception types indicate something is probably wrong
@@ -1988,16 +2031,34 @@ public abstract class Server {
   throws IOException {
     responseBuf.reset();
     DataOutputStream out = new DataOutputStream(responseBuf);
-    RpcResponseHeaderProto.Builder response =  
+    RpcResponseHeaderProto.Builder headerBuilder =  
         RpcResponseHeaderProto.newBuilder();
-    response.setCallId(call.callId);
-    response.setStatus(status);
-
+    headerBuilder.setCallId(call.callId);
+    headerBuilder.setStatus(status);
+    headerBuilder.setServerIpcVersionNum(Server.CURRENT_VERSION);
 
     if (status == RpcStatusProto.SUCCESS) {
+      RpcResponseHeaderProto header = headerBuilder.build();
+      final int headerLen = header.getSerializedSize();
+      int fullLength  = CodedOutputStream.computeRawVarint32Size(headerLen) +
+          headerLen;
       try {
-        response.build().writeDelimitedTo(out);
-        rv.write(out);
+        if (rv instanceof ProtobufRpcEngine.RpcWrapper) {
+          ProtobufRpcEngine.RpcWrapper resWrapper = 
+              (ProtobufRpcEngine.RpcWrapper) rv;
+          fullLength += resWrapper.getLength();
+          out.writeInt(fullLength);
+          header.writeDelimitedTo(out);
+          rv.write(out);
+        } else { // Have to serialize to buffer to get len
+          final DataOutputBuffer buf = new DataOutputBuffer();
+          rv.write(buf);
+          byte[] data = buf.getData();
+          fullLength += buf.getLength();
+          out.writeInt(fullLength);
+          header.writeDelimitedTo(out);
+          out.write(data, 0, buf.getLength());
+        }
       } catch (Throwable t) {
         LOG.warn("Error serializing call response for call " + call, t);
         // Call back to same function - this is OK since the
@@ -2008,13 +2069,15 @@ public abstract class Server {
             StringUtils.stringifyException(t));
         return;
       }
-    } else {
-      if (status == RpcStatusProto.FATAL) {
-        response.setServerIpcVersionNum(Server.CURRENT_VERSION);
-      }
-      response.build().writeDelimitedTo(out);
-      WritableUtils.writeString(out, errorClass);
-      WritableUtils.writeString(out, error);
+    } else { // Rpc Failure
+      headerBuilder.setExceptionClassName(errorClass);
+      headerBuilder.setErrorMsg(error);
+      RpcResponseHeaderProto header = headerBuilder.build();
+      int headerLen = header.getSerializedSize();
+      final int fullLength  = 
+          CodedOutputStream.computeRawVarint32Size(headerLen) + headerLen;
+      out.writeInt(fullLength);
+      header.writeDelimitedTo(out);
     }
     if (call.connection.useWrap) {
       wrapWithSasl(responseBuf, call);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java

@@ -140,10 +140,12 @@ public class MetricsServlet extends HttpServlet {
    */
   void printMap(PrintWriter out, Map<String, Map<String, List<TagsMetricsPair>>> map) {
     for (Map.Entry<String, Map<String, List<TagsMetricsPair>>> context : map.entrySet()) {
-      out.println(context.getKey());
+      out.print(context.getKey());
+      out.print("\n");
       for (Map.Entry<String, List<TagsMetricsPair>> record : context.getValue().entrySet()) {
         indent(out, 1);
-        out.println(record.getKey());
+        out.print(record.getKey());
+        out.print("\n");
         for (TagsMetricsPair pair : record.getValue()) {
           indent(out, 2);
           // Prints tag values in the form "{key=value,key=value}:"
@@ -159,7 +161,7 @@ public class MetricsServlet extends HttpServlet {
             out.print("=");
             out.print(tagValue.getValue().toString());
           }
-          out.println("}:");
+          out.print("}:\n");
           
           // Now print metric values, one per line
           for (Map.Entry<String, Number> metricValue : 
@@ -167,7 +169,8 @@ public class MetricsServlet extends HttpServlet {
             indent(out, 3);
             out.print(metricValue.getKey());
             out.print("=");
-            out.println(metricValue.getValue().toString());
+            out.print(metricValue.getValue().toString());
+            out.print("\n");
           }
         }
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java

@@ -149,4 +149,9 @@ public class CachedDNSToSwitchMapping extends AbstractDNSToSwitchMapping {
   public boolean isSingleSwitch() {
     return isMappingSingleSwitch(rawMapping);
   }
+  
+  @Override
+  public void reloadCachedMappings() {
+    cache.clear();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java

@@ -89,7 +89,12 @@ public class DNS {
       ictx.close();
     }
 
-    return attribute.get("PTR").get().toString();
+    String hostname = attribute.get("PTR").get().toString();
+    int hostnameLength = hostname.length();
+    if (hostname.charAt(hostnameLength - 1) == '.') {
+      hostname = hostname.substring(0, hostnameLength - 1);
+    }
+    return hostname;
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java

@@ -51,4 +51,12 @@ public interface DNSToSwitchMapping {
    * If <i>names</i> is empty, the returned list is also empty
    */
   public List<String> resolve(List<String> names);
+
+  /**
+   * Reload all of the cached mappings.
+   *
+   * If there is a cache, this method will clear it, so that future accesses
+   * will get a chance to see the new data.
+   */
+  public void reloadCachedMappings();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java

@@ -392,8 +392,16 @@ public class NetworkTopology {
       throw new IllegalArgumentException(
         "Not allow to add an inner node: "+NodeBase.getPath(node));
     }
+    int newDepth = NodeBase.locationToDepth(node.getNetworkLocation()) + 1;
     netlock.writeLock().lock();
     try {
+      if ((depthOfAllLeaves != -1) && (depthOfAllLeaves != newDepth)) {
+        LOG.error("Error: can't add leaf node at depth " +
+            newDepth + " to topology:\n" + oldTopoStr);
+        throw new InvalidTopologyException("Invalid network topology. " +
+            "You cannot have a rack and a non-rack node at the same " +
+            "level of the network topology.");
+      }
       Node rack = getNodeForNetworkLocation(node);
       if (rack != null && !(rack instanceof InnerNode)) {
         throw new IllegalArgumentException("Unexpected data node " 
@@ -408,14 +416,6 @@ public class NetworkTopology {
         if (!(node instanceof InnerNode)) {
           if (depthOfAllLeaves == -1) {
             depthOfAllLeaves = node.getLevel();
-          } else {
-            if (depthOfAllLeaves != node.getLevel()) {
-              LOG.error("Error: can't add leaf node at depth " +
-                  node.getLevel() + " to topology:\n" + oldTopoStr);
-              throw new InvalidTopologyException("Invalid network topology. " +
-                  "You cannot have a rack and a non-rack node at the same " +
-                  "level of the network topology.");
-            }
           }
         }
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java

@@ -167,4 +167,16 @@ public class NodeBase implements Node {
   public void setLevel(int level) {
     this.level = level;
   }
+  
+  public static int locationToDepth(String location) {
+    String normalizedLocation = normalize(location);
+    int length = normalizedLocation.length();
+    int depth = 0;
+    for (int i = 0; i < length; i++) {
+      if (normalizedLocation.charAt(i) == PATH_SEPARATOR) {
+        depth++;
+      }
+    }
+    return depth;
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java

@@ -263,5 +263,11 @@ public final class ScriptBasedMapping extends CachedDNSToSwitchMapping {
     public String toString() {
       return scriptName != null ? ("script " + scriptName) : NO_SCRIPT;
     }
+
+    @Override
+    public void reloadCachedMappings() {
+      // Nothing to do here, since RawScriptBasedMapping has no cache, and
+      // does not inherit from CachedDNSToSwitchMapping
+    }
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java

@@ -76,20 +76,24 @@ public class TableMapping extends CachedDNSToSwitchMapping {
     getRawMapping().setConf(conf);
   }
   
+  @Override
+  public void reloadCachedMappings() {
+    super.reloadCachedMappings();
+    getRawMapping().reloadCachedMappings();
+  }
+  
   private static final class RawTableMapping extends Configured
       implements DNSToSwitchMapping {
     
-    private final Map<String, String> map = new HashMap<String, String>();
-    private boolean initialized = false;
+    private Map<String, String> map;
   
-    private synchronized void load() {
-      map.clear();
+    private Map<String, String> load() {
+      Map<String, String> loadMap = new HashMap<String, String>();
   
       String filename = getConf().get(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, null);
       if (StringUtils.isBlank(filename)) {
-        LOG.warn(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY + " not configured. "
-            + NetworkTopology.DEFAULT_RACK + " will be returned.");
-        return;
+        LOG.warn(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY + " not configured. ");
+        return null;
       }
   
       BufferedReader reader = null;
@@ -101,7 +105,7 @@ public class TableMapping extends CachedDNSToSwitchMapping {
           if (line.length() != 0 && line.charAt(0) != '#') {
             String[] columns = line.split("\\s+");
             if (columns.length == 2) {
-              map.put(columns[0], columns[1]);
+              loadMap.put(columns[0], columns[1]);
             } else {
               LOG.warn("Line does not have two columns. Ignoring. " + line);
             }
@@ -109,29 +113,31 @@ public class TableMapping extends CachedDNSToSwitchMapping {
           line = reader.readLine();
         }
       } catch (Exception e) {
-        LOG.warn(filename + " cannot be read. " + NetworkTopology.DEFAULT_RACK
-            + " will be returned.", e);
-        map.clear();
+        LOG.warn(filename + " cannot be read.", e);
+        return null;
       } finally {
         if (reader != null) {
           try {
             reader.close();
           } catch (IOException e) {
-            LOG.warn(filename + " cannot be read. "
-                + NetworkTopology.DEFAULT_RACK + " will be returned.", e);
-            map.clear();
+            LOG.warn(filename + " cannot be read.", e);
+            return null;
           }
         }
       }
+      return loadMap;
     }
   
     @Override
     public synchronized List<String> resolve(List<String> names) {
-      if (!initialized) {
-        initialized = true;
-        load();
+      if (map == null) {
+        map = load();
+        if (map == null) {
+          LOG.warn("Failed to read topology table. " +
+            NetworkTopology.DEFAULT_RACK + " will be used for all nodes.");
+          map = new HashMap<String, String>();
+        }
       }
-  
       List<String> results = new ArrayList<String>(names.size());
       for (String name : names) {
         String result = map.get(name);
@@ -143,6 +149,18 @@ public class TableMapping extends CachedDNSToSwitchMapping {
       }
       return results;
     }
-    
+
+    @Override
+    public void reloadCachedMappings() {
+      Map<String, String> newMap = load();
+      if (newMap == null) {
+        LOG.error("Failed to reload the topology table.  The cached " +
+            "mappings will not be cleared.");
+      } else {
+        synchronized(this) {
+          map = newMap;
+        }
+      }
+    }
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java

@@ -192,7 +192,7 @@ public class Buffer implements Comparable, Cloneable {
     int hash = 1;
     byte[] b = this.get();
     for (int i = 0; i < count; i++)
-      hash = (31 * hash) + (int)b[i];
+      hash = (31 * hash) + b[i];
     return hash;
   }
   

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java

@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * Interface that alll the serializers have to implement.
+ * Interface that all the serializers have to implement.
  * 
  * @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
  */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.security;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
+
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -25,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 /**
  * This class implements parsing and handling of Kerberos principal names. In 
  * particular, it splits them apart and translates them down into local
@@ -36,15 +37,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 @InterfaceStability.Evolving
 public class HadoopKerberosName extends KerberosName {
 
-  static {
-    try {
-      KerberosUtil.getDefaultRealm();
-    } catch (Exception ke) {
-      if(UserGroupInformation.isSecurityEnabled())
-        throw new IllegalArgumentException("Can't get Kerberos configuration",ke);
-    }
-  }
-
   /**
    * Create a name from the full Kerberos principal name.
    * @param name
@@ -63,7 +55,23 @@ public class HadoopKerberosName extends KerberosName {
    * @throws IOException
    */
   public static void setConfiguration(Configuration conf) throws IOException {
-    String ruleString = conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT");
+    final String defaultRule;
+    switch (SecurityUtil.getAuthenticationMethod(conf)) {
+      case KERBEROS:
+      case KERBEROS_SSL:
+        try {
+          KerberosUtil.getDefaultRealm();
+        } catch (Exception ke) {
+          throw new IllegalArgumentException("Can't get Kerberos realm", ke);
+        }
+        defaultRule = "DEFAULT";
+        break;
+      default:
+        // just extract the simple user name
+        defaultRule = "RULE:[1:$1] RULE:[2:$1]";
+        break; 
+    }
+    String ruleString = conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, defaultRule);
     setRules(ruleString);
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java

@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.Hashtable;
 import java.util.List;
 
+import javax.naming.CommunicationException;
 import javax.naming.Context;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
@@ -166,6 +167,8 @@ public class LdapGroupsMapping
   private String groupMemberAttr;
   private String groupNameAttr;
 
+  public static int RECONNECT_RETRY_COUNT = 3;
+  
   /**
    * Returns list of groups for a user.
    * 
@@ -178,34 +181,63 @@ public class LdapGroupsMapping
    */
   @Override
   public synchronized List<String> getGroups(String user) throws IOException {
+    List<String> emptyResults = new ArrayList<String>();
+    /*
+     * Normal garbage collection takes care of removing Context instances when they are no longer in use. 
+     * Connections used by Context instances being garbage collected will be closed automatically.
+     * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. 
+     */
+    try {
+      return doGetGroups(user);
+    } catch (CommunicationException e) {
+      LOG.warn("Connection is closed, will try to reconnect");
+    } catch (NamingException e) {
+      LOG.warn("Exception trying to get groups for user " + user, e);
+      return emptyResults;
+    }
+
+    int retryCount = 0;
+    while (retryCount ++ < RECONNECT_RETRY_COUNT) {
+      //reset ctx so that new DirContext can be created with new connection
+      this.ctx = null;
+      
+      try {
+        return doGetGroups(user);
+      } catch (CommunicationException e) {
+        LOG.warn("Connection being closed, reconnecting failed, retryCount = " + retryCount);
+      } catch (NamingException e) {
+        LOG.warn("Exception trying to get groups for user " + user, e);
+        return emptyResults;
+      }
+    }
+    
+    return emptyResults;
+  }
+  
+  List<String> doGetGroups(String user) throws NamingException {
     List<String> groups = new ArrayList<String>();
 
-    try {
-      DirContext ctx = getDirContext();
-
-      // Search for the user. We'll only ever need to look at the first result
-      NamingEnumeration<SearchResult> results = ctx.search(baseDN,
-                                                           userSearchFilter,
-                                                           new Object[]{user},
-                                                           SEARCH_CONTROLS);
-      if (results.hasMoreElements()) {
-        SearchResult result = results.nextElement();
-        String userDn = result.getNameInNamespace();
-
-        NamingEnumeration<SearchResult> groupResults =
+    DirContext ctx = getDirContext();
+
+    // Search for the user. We'll only ever need to look at the first result
+    NamingEnumeration<SearchResult> results = ctx.search(baseDN,
+        userSearchFilter,
+        new Object[]{user},
+        SEARCH_CONTROLS);
+    if (results.hasMoreElements()) {
+      SearchResult result = results.nextElement();
+      String userDn = result.getNameInNamespace();
+
+      NamingEnumeration<SearchResult> groupResults =
           ctx.search(baseDN,
-                     "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
-                     new Object[]{userDn},
-                     SEARCH_CONTROLS);
-        while (groupResults.hasMoreElements()) {
-          SearchResult groupResult = groupResults.nextElement();
-          Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
-          groups.add(groupName.get().toString());
-        }
+              "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
+              new Object[]{userDn},
+              SEARCH_CONTROLS);
+      while (groupResults.hasMoreElements()) {
+        SearchResult groupResult = groupResults.nextElement();
+        Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
+        groups.add(groupName.get().toString());
       }
-    } catch (NamingException e) {
-      LOG.warn("Exception trying to get groups for user " + user, e);
-      return new ArrayList<String>();
     }
 
     return groups;
@@ -236,7 +268,7 @@ public class LdapGroupsMapping
 
     return ctx;
   }
-
+  
   /**
    * Caches groups, no need to do that for this provider
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java

@@ -30,6 +30,7 @@ import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.ServiceLoader;
 import java.util.Set;
 
@@ -219,7 +220,7 @@ public class SecurityUtil {
     if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) {
       fqdn = getLocalHostName();
     }
-    return components[0] + "/" + fqdn.toLowerCase() + "@" + components[2];
+    return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2];
   }
   
   static String getLocalHostName() throws UnknownHostException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java

@@ -86,7 +86,8 @@ public class ShellBasedUnixGroupsMapping
       LOG.warn("got exception trying to get groups for user " + user, e);
     }
     
-    StringTokenizer tokenizer = new StringTokenizer(result);
+    StringTokenizer tokenizer =
+        new StringTokenizer(result, Shell.TOKEN_SEPARATOR_REGEX);
     List<String> groups = new LinkedList<String>();
     while (tokenizer.hasMoreTokens()) {
       groups.add(tokenizer.nextToken());

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -53,20 +53,20 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
-import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Time;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * User and group information for Hadoop.
  * This class wraps around a JAAS Subject and provides methods to determine the
@@ -190,8 +190,6 @@ public class UserGroupInformation {
 
   /** Metrics to track UGI activity */
   static UgiMetrics metrics = UgiMetrics.create();
-  /** Are the static variables that depend on configuration initialized? */
-  private static boolean isInitialized = false;
   /** The auth method to use */
   private static AuthenticationMethod authenticationMethod;
   /** Server-side groups fetching service */
@@ -211,8 +209,8 @@ public class UserGroupInformation {
    * Must be called before useKerberos or groups is used.
    */
   private static synchronized void ensureInitialized() {
-    if (!isInitialized) {
-        initialize(new Configuration(), KerberosName.hasRulesBeenSet());
+    if (conf == null) {
+      initialize(new Configuration(), false);
     }
   }
 
@@ -220,25 +218,17 @@ public class UserGroupInformation {
    * Initialize UGI and related classes.
    * @param conf the configuration to use
    */
-  private static synchronized void initialize(Configuration conf, boolean skipRulesSetting) {
-    initUGI(conf);
-    // give the configuration on how to translate Kerberos names
-    try {
-      if (!skipRulesSetting) {
+  private static synchronized void initialize(Configuration conf,
+                                              boolean overrideNameRules) {
+    authenticationMethod = SecurityUtil.getAuthenticationMethod(conf);
+    if (overrideNameRules || !HadoopKerberosName.hasRulesBeenSet()) {
+      try {
         HadoopKerberosName.setConfiguration(conf);
+      } catch (IOException ioe) {
+        throw new RuntimeException(
+            "Problem with Kerberos auth_to_local name configuration", ioe);
       }
-    } catch (IOException ioe) {
-      throw new RuntimeException("Problem with Kerberos auth_to_local name " +
-          "configuration", ioe);
     }
-  }
-  
-  /**
-   * Set the configuration values for UGI.
-   * @param conf the configuration to use
-   */
-  private static synchronized void initUGI(Configuration conf) {
-    authenticationMethod = SecurityUtil.getAuthenticationMethod(conf);
     try {
         kerberosMinSecondsBeforeRelogin = 1000L * conf.getLong(
                 HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN,
@@ -253,7 +243,6 @@ public class UserGroupInformation {
     if (!(groups instanceof TestingGroups)) {
       groups = Groups.getUserToGroupsMappingService(conf);
     }
-    isInitialized = true;
     UserGroupInformation.conf = conf;
   }
 
@@ -266,7 +255,18 @@ public class UserGroupInformation {
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
   public static void setConfiguration(Configuration conf) {
-    initialize(conf, false);
+    initialize(conf, true);
+  }
+  
+  @InterfaceAudience.Private
+  @VisibleForTesting
+  static void reset() {
+    authenticationMethod = null;
+    conf = null;
+    groups = null;
+    kerberosMinSecondsBeforeRelogin = 0;
+    setLoginUser(null);
+    HadoopKerberosName.setRules(null);
   }
   
   /**
@@ -713,7 +713,8 @@ public class UserGroupInformation {
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
-  synchronized static void setLoginUser(UserGroupInformation ugi) {
+  @VisibleForTesting
+  public synchronized static void setLoginUser(UserGroupInformation ugi) {
     // if this is to become stable, should probably logout the currently
     // logged in ugi if it's different
     loginUser = ugi;
@@ -1498,7 +1499,7 @@ public class UserGroupInformation {
       } else if (cause instanceof InterruptedException) {
         throw (InterruptedException) cause;
       } else {
-        throw new UndeclaredThrowableException(pae,"Unknown exception in doAs");
+        throw new UndeclaredThrowableException(cause);
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java

@@ -164,7 +164,9 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
     } else {
       keystore.load(null, null);
     }
-    KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance("SunX509");
+    KeyManagerFactory keyMgrFactory = KeyManagerFactory
+        .getInstance(SSLFactory.SSLCERTIFICATE);
+      
     keyMgrFactory.init(keystore, (keystorePassword != null) ?
                                  keystorePassword.toCharArray() : null);
     keyManagers = keyMgrFactory.getKeyManagers();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java

@@ -169,8 +169,8 @@ public final class ReloadingX509TrustManager
       in.close();
     }
 
-    TrustManagerFactory trustManagerFactory =
-      TrustManagerFactory.getInstance("SunX509");
+    TrustManagerFactory trustManagerFactory = 
+      TrustManagerFactory.getInstance(SSLFactory.SSLCERTIFICATE);
     trustManagerFactory.init(ks);
     TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
     for (TrustManager trustManager1 : trustManagers) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java

@@ -58,6 +58,9 @@ public class SSLFactory implements ConnectionConfigurator {
     "hadoop.ssl.client.conf";
   public static final String SSL_SERVER_CONF_KEY =
     "hadoop.ssl.server.conf";
+  private static final boolean IBMJAVA = 
+      System.getProperty("java.vendor").contains("IBM");
+  public static final String SSLCERTIFICATE = IBMJAVA?"ibmX509":"SunX509"; 
 
   public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false;
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.Enumeration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class ClassUtil {
+  /** 
+   * Find a jar that contains a class of the same name, if any.
+   * It will return a jar file, even if that is not the first thing
+   * on the class path that has a class with the same name.
+   * 
+   * @param clazz the class to find.
+   * @return a jar file that contains the class, or null.
+   * @throws IOException
+   */
+  public static String findContainingJar(Class clazz) {
+    ClassLoader loader = clazz.getClassLoader();
+    String classFile = clazz.getName().replaceAll("\\.", "/") + ".class";
+    try {
+      for (Enumeration itr = loader.getResources(classFile);
+          itr.hasMoreElements();) {
+        URL url = (URL) itr.nextElement();
+        if ("jar".equals(url.getProtocol())) {
+          String toReturn = url.getPath();
+          if (toReturn.startsWith("file:")) {
+            toReturn = toReturn.substring("file:".length());
+          }
+          toReturn = URLDecoder.decode(toReturn, "UTF-8");
+          return toReturn.replaceAll("!.*$", "");
+        }
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return null;
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java

@@ -23,11 +23,10 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
 
 /**
  * Class that provides utility functions for checking disk problem
@@ -36,10 +35,16 @@ import org.apache.hadoop.fs.permission.FsPermission;
 @InterfaceStability.Unstable
 public class DiskChecker {
 
+  private static final long SHELL_TIMEOUT = 10 * 1000;
+
   public static class DiskErrorException extends IOException {
     public DiskErrorException(String msg) {
       super(msg);
     }
+
+    public DiskErrorException(String msg, Throwable cause) {
+      super(msg, cause);
+    }
   }
     
   public static class DiskOutOfSpaceException extends IOException {
@@ -85,25 +90,11 @@ public class DiskChecker {
    * @throws DiskErrorException
    */
   public static void checkDir(File dir) throws DiskErrorException {
-    if (!mkdirsWithExistsCheck(dir))
+    if (!mkdirsWithExistsCheck(dir)) {
       throw new DiskErrorException("Can not create directory: "
                                    + dir.toString());
-
-    if (!dir.isDirectory())
-      throw new DiskErrorException("Not a directory: "
-                                   + dir.toString());
-
-    if (!dir.canRead())
-      throw new DiskErrorException("Directory is not readable: "
-                                   + dir.toString());
-
-    if (!dir.canWrite())
-      throw new DiskErrorException("Directory is not writable: "
-                                   + dir.toString());
-
-    if (!dir.canExecute())
-      throw new DiskErrorException("Directory is not executable: "
-	  + dir.toString());
+    }
+    checkDirAccess(dir);
   }
 
   /**
@@ -152,24 +143,102 @@ public class DiskChecker {
                               FsPermission expected)
   throws DiskErrorException, IOException {
     mkdirsWithExistsAndPermissionCheck(localFS, dir, expected);
+    checkDirAccess(localFS.pathToFile(dir));
+  }
 
-    FileStatus stat = localFS.getFileStatus(dir);
-    FsPermission actual = stat.getPermission();
+  /**
+   * Checks that the given file is a directory and that the current running
+   * process can read, write, and execute it.
+   * 
+   * @param dir File to check
+   * @throws DiskErrorException if dir is not a directory, not readable, not
+   *   writable, or not executable
+   */
+  private static void checkDirAccess(File dir) throws DiskErrorException {
+    if (!dir.isDirectory()) {
+      throw new DiskErrorException("Not a directory: "
+                                   + dir.toString());
+    }
 
-    if (!stat.isDirectory())
-      throw new DiskErrorException("not a directory: "+ dir.toString());
+    if (Shell.WINDOWS) {
+      checkAccessByFileSystemInteraction(dir);
+    } else {
+      checkAccessByFileMethods(dir);
+    }
+  }
 
-    FsAction user = actual.getUserAction();
-    if (!user.implies(FsAction.READ))
-      throw new DiskErrorException("directory is not readable: "
+  /**
+   * Checks that the current running process can read, write, and execute the
+   * given directory by using methods of the File object.
+   * 
+   * @param dir File to check
+   * @throws DiskErrorException if dir is not readable, not writable, or not
+   *   executable
+   */
+  private static void checkAccessByFileMethods(File dir)
+      throws DiskErrorException {
+    if (!dir.canRead()) {
+      throw new DiskErrorException("Directory is not readable: "
+                                   + dir.toString());
+    }
+
+    if (!dir.canWrite()) {
+      throw new DiskErrorException("Directory is not writable: "
                                    + dir.toString());
+    }
 
-    if (!user.implies(FsAction.WRITE))
-      throw new DiskErrorException("directory is not writable: "
+    if (!dir.canExecute()) {
+      throw new DiskErrorException("Directory is not executable: "
                                    + dir.toString());
+    }
+  }
 
-    if (!user.implies(FsAction.EXECUTE))
-      throw new DiskErrorException("directory is not listable: "
+  /**
+   * Checks that the current running process can read, write, and execute the
+   * given directory by attempting each of those operations on the file system.
+   * This method contains several workarounds to known JVM bugs that cause
+   * File.canRead, File.canWrite, and File.canExecute to return incorrect results
+   * on Windows with NTFS ACLs.  See:
+   * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6203387
+   * These bugs are supposed to be fixed in JDK7.
+   * 
+   * @param dir File to check
+   * @throws DiskErrorException if dir is not readable, not writable, or not
+   *   executable
+   */
+  private static void checkAccessByFileSystemInteraction(File dir)
+      throws DiskErrorException {
+    // Make sure we can read the directory by listing it.
+    if (dir.list() == null) {
+      throw new DiskErrorException("Directory is not readable: "
                                    + dir.toString());
+    }
+
+    // Make sure we can write to the directory by creating a temp file in it.
+    try {
+      File tempFile = File.createTempFile("checkDirAccess", null, dir);
+      if (!tempFile.delete()) {
+        throw new DiskErrorException("Directory is not writable: "
+                                     + dir.toString());
+      }
+    } catch (IOException e) {
+      throw new DiskErrorException("Directory is not writable: "
+                                   + dir.toString(), e);
+    }
+
+    // Make sure the directory is executable by trying to cd into it.  This
+    // launches a separate process.  It does not change the working directory of
+    // the current process.
+    try {
+      String[] cdCmd = new String[] { "cmd", "/C", "cd",
+          dir.getAbsolutePath() };
+      Shell.execCommand(null, cdCmd, SHELL_TIMEOUT);
+    } catch (Shell.ExitCodeException e) {
+      throw new DiskErrorException("Directory is not executable: "
+                                   + dir.toString(), e);
+    } catch (IOException e) {
+      throw new DiskErrorException("Directory is not executable: "
+                                   + dir.toString(), e);
+    }
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java

@@ -42,6 +42,8 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
 
 /**
  * <code>GenericOptionsParser</code> is a utility to parse command line
@@ -321,15 +323,17 @@ public class GenericOptionsParser {
       String fileName = line.getOptionValue("tokenCacheFile");
       // check if the local file exists
       FileSystem localFs = FileSystem.getLocal(conf);
-      Path p = new Path(fileName);
+      Path p = localFs.makeQualified(new Path(fileName));
       if (!localFs.exists(p)) {
           throw new FileNotFoundException("File "+fileName+" does not exist.");
       }
       if(LOG.isDebugEnabled()) {
         LOG.debug("setting conf tokensFile: " + fileName);
       }
-      conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p)
-          .toString(), "from -tokenCacheFile command line option");
+      UserGroupInformation.getCurrentUser().addCredentials(
+          Credentials.readTokenStorageFile(p, conf));
+      conf.set("mapreduce.job.credentials.json", p.toString(),
+               "from -tokenCacheFile command line option");
 
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java

@@ -60,7 +60,7 @@ class NativeCrc32 {
         fileName, basePos);
   }
   
-  private static native void nativeVerifyChunkedSums(
+    private static native void nativeVerifyChunkedSums(
       int bytesPerSum, int checksumType,
       ByteBuffer sums, int sumsOffset,
       ByteBuffer data, int dataOffset, int dataLength,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java

@@ -32,9 +32,10 @@ public class PlatformName {
    * The complete platform 'name' to identify the platform as 
    * per the java-vm.
    */
-  private static final String platformName = System.getProperty("os.name") + "-" + 
-    System.getProperty("os.arch") + "-" +
-    System.getProperty("sun.arch.data.model");
+  private static final String platformName =
+      (Shell.WINDOWS ? System.getenv("os") : System.getProperty("os.name"))
+      + "-" + System.getProperty("os.arch")
+      + "-" + System.getProperty("sun.arch.data.model");
   
   /**
    * Get the complete platform as per the java-vm.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

@@ -21,6 +21,7 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
@@ -44,46 +45,208 @@ abstract public class Shell {
   
   public static final Log LOG = LogFactory.getLog(Shell.class);
   
+  private static boolean IS_JAVA7_OR_ABOVE =
+      System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
+
+  public static boolean isJava7OrAbove() {
+    return IS_JAVA7_OR_ABOVE;
+  }
+
   /** a Unix command to get the current user's name */
   public final static String USER_NAME_COMMAND = "whoami";
+
+  /** Windows CreateProcess synchronization object */
+  public static final Object WindowsProcessLaunchLock = new Object();
+
   /** a Unix command to get the current user's groups list */
   public static String[] getGroupsCommand() {
-    return new String[]{"bash", "-c", "groups"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
+                    : new String[]{"bash", "-c", "groups"};
   }
+
   /** a Unix command to get a given user's groups list */
   public static String[] getGroupsForUserCommand(final String user) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "id -Gn " + user};
+    return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
+                    : new String [] {"bash", "-c", "id -Gn " + user};
   }
+
   /** a Unix command to get a given netgroup's user list */
   public static String[] getUsersForNetgroupCommand(final String netgroup) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "getent netgroup " + netgroup};
+    return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
+                    : new String [] {"bash", "-c", "getent netgroup " + netgroup};
+  }
+
+  /** Return a command to get permission information. */
+  public static String[] getGetPermissionCommand() {
+    return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
+                     : new String[] { "/bin/ls", "-ld" };
+  }
+
+  /** Return a command to set permission */
+  public static String[] getSetPermissionCommand(String perm, boolean recursive) {
+    if (recursive) {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
+                         : new String[] { "chmod", "-R", perm };
+    } else {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
+                       : new String[] { "chmod", perm };
+    }
   }
+
+  /**
+   * Return a command to set permission for specific file.
+   * 
+   * @param perm String permission to set
+   * @param recursive boolean true to apply to all sub-directories recursively
+   * @param file String file to set
+   * @return String[] containing command and arguments
+   */
+  public static String[] getSetPermissionCommand(String perm, boolean recursive,
+                                                 String file) {
+    String[] baseCmd = getSetPermissionCommand(perm, recursive);
+    String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
+    cmdWithFile[cmdWithFile.length - 1] = file;
+    return cmdWithFile;
+  }
+
+  /** Return a command to set owner */
+  public static String[] getSetOwnerCommand(String owner) {
+    return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
+                     : new String[] { "chown", owner };
+  }
+  
+  /** Return a command to create symbolic links */
+  public static String[] getSymlinkCommand(String target, String link) {
+    return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
+                   : new String[] { "ln", "-s", target, link };
+  }
+
   /** a Unix command to set permission */
   public static final String SET_PERMISSION_COMMAND = "chmod";
   /** a Unix command to set owner */
   public static final String SET_OWNER_COMMAND = "chown";
+
+  /** a Unix command to set the change user's groups list */
   public static final String SET_GROUP_COMMAND = "chgrp";
   /** a Unix command to create a link */
   public static final String LINK_COMMAND = "ln";
   /** a Unix command to get a link target */
   public static final String READ_LINK_COMMAND = "readlink";
-  /** Return a Unix command to get permission information. */
-  public static String[] getGET_PERMISSION_COMMAND() {
-    //force /bin/ls, except on windows.
-    return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"};
-  }
 
   /**Time after which the executing script would be timedout*/
   protected long timeOutInterval = 0L;
   /** If or not script timed out*/
   private AtomicBoolean timedOut;
 
+
+  /** Centralized logic to discover and validate the sanity of the Hadoop 
+   *  home directory. Returns either NULL or a directory that exists and 
+   *  was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV 
+   *  variable.  This does a lot of work so it should only be called 
+   *  privately for initialization once per process.
+   **/
+  private static String checkHadoopHome() {
+
+    // first check the Dflag hadoop.home.dir with JVM scope
+    String home = System.getProperty("hadoop.home.dir");
+
+    // fall back to the system/user-global env variable
+    if (home == null) {
+      home = System.getenv("HADOOP_HOME");
+    }
+
+    try {
+       // couldn't find either setting for hadoop's home directory
+       if (home == null) {
+         throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
+       }
+
+       if (home.startsWith("\"") && home.endsWith("\"")) {
+         home = home.substring(1, home.length()-1);
+       }
+
+       // check that the home setting is actually a directory that exists
+       File homedir = new File(home);
+       if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
+         throw new IOException("Hadoop home directory " + homedir
+           + " does not exist, is not a directory, or is not an absolute path.");
+       }
+
+       home = homedir.getCanonicalPath();
+
+    } catch (IOException ioe) {
+       LOG.error("Failed to detect a valid hadoop home directory", ioe);
+       home = null;
+    }
+    
+    return home;
+  }
+  private static String HADOOP_HOME_DIR = checkHadoopHome();
+
+  // Public getter, throws an exception if HADOOP_HOME failed validation
+  // checks and is being referenced downstream.
+  public static final String getHadoopHome() throws IOException {
+    if (HADOOP_HOME_DIR == null) {
+      throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
+    }
+
+    return HADOOP_HOME_DIR;
+  }
+
+  /** fully qualify the path to a binary that should be in a known hadoop 
+   *  bin location. This is primarily useful for disambiguating call-outs 
+   *  to executable sub-components of Hadoop to avoid clashes with other 
+   *  executables that may be in the path.  Caveat:  this call doesn't 
+   *  just format the path to the bin directory.  It also checks for file 
+   *  existence of the composed path. The output of this call should be 
+   *  cached by callers.
+   * */
+  public static final String getQualifiedBinPath(String executable) 
+  throws IOException {
+    // construct hadoop bin path to the specified executable
+    String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" 
+      + File.separator + executable;
+
+    File exeFile = new File(fullExeName);
+    if (!exeFile.exists()) {
+      throw new IOException("Could not locate executable " + fullExeName
+        + " in the Hadoop binaries.");
+    }
+
+    return exeFile.getCanonicalPath();
+  }
+
   /** Set to true on Windows platforms */
   public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
                 = System.getProperty("os.name").startsWith("Windows");
+
+  public static final boolean LINUX
+                = System.getProperty("os.name").startsWith("Linux");
   
+  /** a Windows utility to emulate Unix commands */
+  public static final String WINUTILS = getWinUtilsPath();
+
+  public static final String getWinUtilsPath() {
+    String winUtilsPath = null;
+
+    try {
+      if (WINDOWS) {
+        winUtilsPath = getQualifiedBinPath("winutils.exe");
+      }
+    } catch (IOException ioe) {
+       LOG.error("Failed to locate the winutils binary in the hadoop binary path",
+         ioe);
+    }
+
+    return winUtilsPath;
+  }
+
+  /** Token separator regex used to parse Shell tool outputs */
+  public static final String TOKEN_SEPARATOR_REGEX
+                = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
+
   private long    interval;   // refresh interval in msec
   private long    lastTime;   // last time the command was performed
   private Map<String, String> environment; // env for the command execution
@@ -144,7 +307,19 @@ abstract public class Shell {
       builder.directory(this.dir);
     }
     
-    process = builder.start();
+    if (Shell.WINDOWS) {
+      synchronized (WindowsProcessLaunchLock) {
+        // To workaround the race condition issue with child processes
+        // inheriting unintended handles during process launch that can
+        // lead to hangs on reading output and error streams, we
+        // serialize process creation. More info available at:
+        // http://support.microsoft.com/kb/315939
+        process = builder.start();
+      }
+    } else {
+      process = builder.start();
+    }
+
     if (timeOutInterval > 0) {
       timeOutTimer = new Timer("Shell command timeout");
       timeoutTimerTask = new ShellTimeoutTimerTask(

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java

@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import sun.misc.Signal;
+import sun.misc.SignalHandler;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This class logs a message whenever we're about to exit on a UNIX signal.
+ * This is helpful for determining the root cause of a process' exit.
+ * For example, if the process exited because the system administrator 
+ * ran a standard "kill," you would see 'EXITING ON SIGNAL SIGTERM' in the log.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum SignalLogger {
+  INSTANCE;
+
+  private boolean registered = false;
+
+  /**
+   * Our signal handler.
+   */
+  private static class Handler implements SignalHandler {
+    final private org.apache.commons.logging.Log LOG;
+    final private SignalHandler prevHandler;
+
+    Handler(String name, Log LOG) {
+      this.LOG = LOG;
+      prevHandler = Signal.handle(new Signal(name), this);
+    }
+
+    /**
+     * Handle an incoming signal.
+     *
+     * @param signal    The incoming signal
+     */
+    @Override
+    public void handle(Signal signal) {
+      LOG.error("RECEIVED SIGNAL " + signal.getNumber() +
+          ": SIG" + signal.getName());
+      prevHandler.handle(signal);
+    }
+  }
+
+  /**
+   * Register some signal handlers.
+   *
+   * @param LOG        The log4j logfile to use in the signal handlers.
+   */
+  public void register(final Log LOG) {
+    if (registered) {
+      throw new IllegalStateException("Can't re-install the signal handlers.");
+    }
+    registered = true;
+    StringBuilder bld = new StringBuilder();
+    bld.append("registered UNIX signal handlers for [");
+    final String SIGNALS[] = { "TERM", "HUP", "INT" };
+    String separator = "";
+    for (String signalName : SIGNALS) {
+      try {
+        new Handler(signalName, LOG);
+        bld.append(separator);
+        bld.append(signalName);
+        separator = ", ";
+      } catch (Exception e) {
+        LOG.debug(e);
+      }
+    }
+    bld.append("]");
+    LOG.info(bld.toString());
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -30,12 +30,17 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.StringTokenizer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import org.apache.commons.lang.SystemUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.Shell;
 
 import com.google.common.net.InetAddresses;
 
@@ -51,6 +56,27 @@ public class StringUtils {
    */
   public static final int SHUTDOWN_HOOK_PRIORITY = 0;
 
+  /**
+   * Shell environment variables: $ followed by one letter or _ followed by
+   * multiple letters, numbers, or underscores.  The group captures the
+   * environment variable name without the leading $.
+   */
+  public static final Pattern SHELL_ENV_VAR_PATTERN =
+    Pattern.compile("\\$([A-Za-z_]{1}[A-Za-z0-9_]*)");
+
+  /**
+   * Windows environment variables: surrounded by %.  The group captures the
+   * environment variable name without the leading and trailing %.
+   */
+  public static final Pattern WIN_ENV_VAR_PATTERN = Pattern.compile("%(.*?)%");
+
+  /**
+   * Regular expression that matches and captures environment variable names
+   * according to platform-specific rules.
+   */
+  public static final Pattern ENV_VAR_PATTERN = Shell.WINDOWS ?
+    WIN_ENV_VAR_PATTERN : SHELL_ENV_VAR_PATTERN;
+
   /**
    * Make a string representation of the exception.
    * @param e The exception to stringify
@@ -588,6 +614,13 @@ public class StringUtils {
         )
       );
 
+    if (SystemUtils.IS_OS_UNIX) {
+      try {
+        SignalLogger.INSTANCE.register(LOG);
+      } catch (Throwable t) {
+        LOG.warn("failed to register any UNIX signal loggers: ", t);
+      }
+    }
     ShutdownHookManager.get().addShutdownHook(
       new Runnable() {
         @Override
@@ -791,6 +824,28 @@ public class StringUtils {
     return sb.toString();
   }
 
+  /**
+   * Concatenates strings, using a separator.
+   *
+   * @param separator to join with
+   * @param strings to join
+   * @return  the joined string
+   */
+  public static String join(CharSequence separator, String[] strings) {
+    // Ideally we don't have to duplicate the code here if array is iterable.
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (String s : strings) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(separator);
+      }
+      sb.append(s);
+    }
+    return sb.toString();
+  }
+
   /**
    * Convert SOME_STUFF to SomeStuff
    *
@@ -806,4 +861,37 @@ public class StringUtils {
 
     return sb.toString();
   }
+
+  /**
+   * Matches a template string against a pattern, replaces matched tokens with
+   * the supplied replacements, and returns the result.  The regular expression
+   * must use a capturing group.  The value of the first capturing group is used
+   * to look up the replacement.  If no replacement is found for the token, then
+   * it is replaced with the empty string.
+   * 
+   * For example, assume template is "%foo%_%bar%_%baz%", pattern is "%(.*?)%",
+   * and replacements contains 2 entries, mapping "foo" to "zoo" and "baz" to
+   * "zaz".  The result returned would be "zoo__zaz".
+   * 
+   * @param template String template to receive replacements
+   * @param pattern Pattern to match for identifying tokens, must use a capturing
+   *   group
+   * @param replacements Map<String, String> mapping tokens identified by the
+   *   capturing group to their replacement values
+   * @return String template with replacements
+   */
+  public static String replaceTokens(String template, Pattern pattern,
+      Map<String, String> replacements) {
+    StringBuffer sb = new StringBuffer();
+    Matcher matcher = pattern.matcher(template);
+    while (matcher.find()) {
+      String replacement = replacements.get(matcher.group(1));
+      if (replacement == null) {
+        replacement = "";
+      }
+      matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+    }
+    matcher.appendTail(sb);
+    return sb.toString();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java

@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.util;
 
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.Enumeration;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -155,5 +160,7 @@ public class VersionInfo {
     System.out.println("Subversion " + getUrl() + " -r " + getRevision());
     System.out.println("Compiled by " + getUser() + " on " + getDate());
     System.out.println("From source with checksum " + getSrcChecksum());
+    System.out.println("This command was run using " + 
+        ClassUtil.findContainingJar(VersionInfo.class));
   }
 }

hadoop-common-project/hadoop-common/src/main/java/overview.html

@@ -60,9 +60,7 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:</
     Hadoop was been demonstrated on GNU/Linux clusters with 2000 nodes.
   </li>
   <li>
-    Win32 is supported as a <i>development</i> platform. Distributed operation 
-    has not been well tested on Win32, so this is not a <i>production</i> 
-    platform.
+    Windows is also a supported platform.
   </li>  
 </ul>
   
@@ -84,15 +82,6 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:</
   </li>
 </ol>
 
-<h4>Additional requirements for Windows</h4>
-
-<ol>
-  <li>
-    <a href="http://www.cygwin.com/">Cygwin</a> - Required for shell support in 
-    addition to the required software above.
-  </li>
-</ol>
-  
 <h3>Installing Required Software</h3>
 
 <p>If your platform does not have the required software listed above, you
@@ -104,13 +93,6 @@ $ sudo apt-get install ssh<br>
 $ sudo apt-get install rsync<br>
 </pre></blockquote></p>
 
-<p>On Windows, if you did not install the required software when you
-installed cygwin, start the cygwin installer and select the packages:</p>
-<ul>
-  <li>openssh - the "Net" category</li>
-  <li>rsync - the "Net" category</li>
-</ul>
-
 <h2>Getting Started</h2>
 
 <p>First, you need to get a copy of the Hadoop code.</p>

hadoop-common-project/hadoop-common/src/main/native/native.sln

@@ -0,0 +1,48 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "native", "native.vcxproj", "{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Mixed Platforms = Debug|Mixed Platforms
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Mixed Platforms = Release|Mixed Platforms
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

hadoop-common-project/hadoop-common/src/main/native/native.vcxproj

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>native</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <OutDir>..\..\..\target\bin\</OutDir>
+    <IntDir>..\..\..\target\native\$(Configuration)\</IntDir>
+    <TargetName>hadoop</TargetName>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <CompileAs>CompileAsC</CompileAs>
+      <DisableSpecificWarnings>4244</DisableSpecificWarnings>
+    </ClCompile>
+    <Link>
+      <SubSystem>Windows</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalDependencies>Ws2_32.lib;libwinutils.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>..\..\..\target\bin;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Compressor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Decompressor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\NativeIO.c" />
+    <ClCompile Include="src\org\apache\hadoop\security\JniBasedUnixGroupsMappingWin.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\bulk_crc32.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCodeLoader.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" />
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h" />
+    <ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\bulk_crc32.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\crc32c_tables.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h" />
+    <ClInclude Include="src\org_apache_hadoop.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Resource Files">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\NativeIO.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\bulk_crc32.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCodeLoader.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Compressor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Decompressor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\security\JniBasedUnixGroupsMappingWin.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\bulk_crc32.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\crc32c_tables.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org_apache_hadoop.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c

@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "org_apache_hadoop_io_compress_bzip2.h"
+#include "org_apache_hadoop_io_compress_bzip2_Bzip2Compressor.h"
+
+static jfieldID Bzip2Compressor_clazz;
+static jfieldID Bzip2Compressor_stream;
+static jfieldID Bzip2Compressor_uncompressedDirectBuf;
+static jfieldID Bzip2Compressor_uncompressedDirectBufOff;
+static jfieldID Bzip2Compressor_uncompressedDirectBufLen;
+static jfieldID Bzip2Compressor_compressedDirectBuf;
+static jfieldID Bzip2Compressor_directBufferSize;
+static jfieldID Bzip2Compressor_finish;
+static jfieldID Bzip2Compressor_finished;
+
+static int (*dlsym_BZ2_bzCompressInit)(bz_stream*, int, int, int);
+static int (*dlsym_BZ2_bzCompress)(bz_stream*, int);
+static int (*dlsym_BZ2_bzCompressEnd)(bz_stream*);
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_initIDs(
+                                 JNIEnv *env, jclass class, jstring libname)
+{
+    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (strcmp(bzlib_name, "system-native") == 0)
+      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    // Load the native library.
+    void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
+    if (!libbz2) {
+        THROW(env, "java/lang/UnsatisfiedLinkError",
+              "Cannot load bzip2 native library");
+        return;
+    }
+
+    // Locate the requisite symbols from libbz2.so.
+    dlerror();                                 // Clear any existing error.
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressInit, env, libbz2,
+                        "BZ2_bzCompressInit");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompress, env, libbz2,
+                        "BZ2_bzCompress");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressEnd, env, libbz2,
+                        "BZ2_bzCompressEnd");
+
+    // Initialize the requisite fieldIds.
+    Bzip2Compressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+                                                     "Ljava/lang/Class;");
+    Bzip2Compressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
+    Bzip2Compressor_finish = (*env)->GetFieldID(env, class, "finish", "Z");
+    Bzip2Compressor_finished = (*env)->GetFieldID(env, class, "finished", "Z");
+    Bzip2Compressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                       "uncompressedDirectBuf",
+                                                       "Ljava/nio/Buffer;");
+    Bzip2Compressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, 
+                                                  "uncompressedDirectBufOff",
+                                                  "I");
+    Bzip2Compressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, 
+                                                  "uncompressedDirectBufLen",
+                                                  "I");
+    Bzip2Compressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                     "compressedDirectBuf", 
+                                                     "Ljava/nio/Buffer;");
+    Bzip2Compressor_directBufferSize = (*env)->GetFieldID(env, class, 
+                                                  "directBufferSize", "I");
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_init(
+            JNIEnv *env, jclass class, jint blockSize, jint workFactor)
+{
+    // Create a bz_stream.
+    bz_stream *stream = malloc(sizeof(bz_stream));
+    if (!stream) {
+        THROW(env, "java/lang/OutOfMemoryError", NULL);
+        return (jlong)0;
+    }
+    memset((void*)stream, 0, sizeof(bz_stream));
+
+    // Initialize stream.
+    int rv = (*dlsym_BZ2_bzCompressInit)(stream, blockSize, 0, workFactor);
+    if (rv != BZ_OK) {
+        // Contingency - Report error by throwing appropriate exceptions.
+        free(stream);
+        stream = NULL;
+        
+        switch (rv) {
+        case BZ_MEM_ERROR: 
+            {
+                THROW(env, "java/lang/OutOfMemoryError", NULL);
+            }
+            break;
+        case BZ_PARAM_ERROR:
+            {
+                THROW(env,
+                      "java/lang/IllegalArgumentException",
+                      NULL);
+            }
+            break;
+        default:
+            {
+                THROW(env, "java/lang/InternalError", NULL);
+            }
+            break;
+        }
+    }
+        
+    return JLONG(stream);
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_deflateBytesDirect(
+        JNIEnv *env, jobject this)
+{
+    // Get members of Bzip2Compressor.
+    bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this, 
+                                              Bzip2Compressor_stream));
+    if (!stream) {
+        THROW(env, "java/lang/NullPointerException", NULL);
+        return (jint)0;
+    } 
+
+    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+                                                 Bzip2Compressor_clazz);
+    jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, 
+                                     Bzip2Compressor_uncompressedDirectBuf);
+    jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, 
+                                   Bzip2Compressor_uncompressedDirectBufOff);
+    jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                   Bzip2Compressor_uncompressedDirectBufLen);
+
+    jobject compressed_direct_buf = (*env)->GetObjectField(env, this, 
+                                   Bzip2Compressor_compressedDirectBuf);
+    jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                 Bzip2Compressor_directBufferSize);
+
+    jboolean finish = (*env)->GetBooleanField(env, this,
+                                              Bzip2Compressor_finish);
+
+    // Get the input and output direct buffers.
+    LOCK_CLASS(env, clazz, "Bzip2Compressor");
+    char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                uncompressed_direct_buf);
+    char* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                compressed_direct_buf);
+    UNLOCK_CLASS(env, clazz, "Bzip2Compressor");
+
+    if (!uncompressed_bytes || !compressed_bytes) {
+        return (jint)0;
+    }
+        
+    // Re-calibrate the bz_stream.
+    stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off;
+    stream->avail_in = uncompressed_direct_buf_len;
+    stream->next_out = compressed_bytes;
+    stream->avail_out = compressed_direct_buf_len;
+        
+    // Compress.
+    int rv = dlsym_BZ2_bzCompress(stream, finish ? BZ_FINISH : BZ_RUN);
+
+    jint no_compressed_bytes = 0;
+    switch (rv) {
+        // Contingency? - Report error by throwing appropriate exceptions.
+    case BZ_STREAM_END:
+        {
+            (*env)->SetBooleanField(env, this,
+                                    Bzip2Compressor_finished,
+                                    JNI_TRUE);
+        } // cascade
+    case BZ_RUN_OK:
+    case BZ_FINISH_OK:
+        {
+            uncompressed_direct_buf_off +=
+                uncompressed_direct_buf_len - stream->avail_in;
+            (*env)->SetIntField(env, this, 
+                                Bzip2Compressor_uncompressedDirectBufOff,
+                                uncompressed_direct_buf_off);
+            (*env)->SetIntField(env, this, 
+                                Bzip2Compressor_uncompressedDirectBufLen,
+                                stream->avail_in);
+            no_compressed_bytes =
+                compressed_direct_buf_len - stream->avail_out;
+        }
+        break;
+    default:
+        {
+            THROW(env, "java/lang/InternalError", NULL);
+        }
+        break;
+    }
+        
+    return no_compressed_bytes;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesRead(
+                            JNIEnv *env, jclass class, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesWritten(
+                                JNIEnv *env, jclass class, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32;
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
+                                JNIEnv *env, jclass class, jlong stream)
+{
+    if (dlsym_BZ2_bzCompressEnd(BZSTREAM(stream)) != BZ_OK) {
+        THROW(env, "java/lang/InternalError", NULL);
+    } else {
+        free(BZSTREAM(stream));
+    }
+}
+
+/**
+ * vim: sw=2: ts=2: et:
+ */
+

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c

@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "org_apache_hadoop_io_compress_bzip2.h"
+#include "org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor.h"
+
+static jfieldID Bzip2Decompressor_clazz;
+static jfieldID Bzip2Decompressor_stream;
+static jfieldID Bzip2Decompressor_compressedDirectBuf;
+static jfieldID Bzip2Decompressor_compressedDirectBufOff;
+static jfieldID Bzip2Decompressor_compressedDirectBufLen;
+static jfieldID Bzip2Decompressor_uncompressedDirectBuf;
+static jfieldID Bzip2Decompressor_directBufferSize;
+static jfieldID Bzip2Decompressor_finished;
+
+static int (*dlsym_BZ2_bzDecompressInit)(bz_stream*, int, int);
+static int (*dlsym_BZ2_bzDecompress)(bz_stream*);
+static int (*dlsym_BZ2_bzDecompressEnd)(bz_stream*);
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_initIDs(
+                                 JNIEnv *env, jclass class, jstring libname)
+{
+    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (strcmp(bzlib_name, "system-native") == 0)
+      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    // Load the native library.
+    void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
+    if (!libbz2) {
+        THROW(env, "java/lang/UnsatisfiedLinkError",
+              "Cannot load bzip2 native library");
+        return;
+    }
+
+    // Locate the requisite symbols from libbz2.so.
+    dlerror();                                 // Clear any existing error.
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressInit, env, libbz2,
+                        "BZ2_bzDecompressInit");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompress, env, libbz2,
+                        "BZ2_bzDecompress");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressEnd, env, libbz2,
+                        "BZ2_bzDecompressEnd");
+
+    // Initialize the requisite fieldIds.
+    Bzip2Decompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+                                                       "Ljava/lang/Class;");
+    Bzip2Decompressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
+    Bzip2Decompressor_finished = (*env)->GetFieldID(env, class,
+                                                    "finished", "Z");
+    Bzip2Decompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                        "compressedDirectBuf", 
+                                                        "Ljava/nio/Buffer;");
+    Bzip2Decompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, 
+                                                "compressedDirectBufOff", "I");
+    Bzip2Decompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, 
+                                                "compressedDirectBufLen", "I");
+    Bzip2Decompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                "uncompressedDirectBuf", 
+                                                "Ljava/nio/Buffer;");
+    Bzip2Decompressor_directBufferSize = (*env)->GetFieldID(env, class, 
+                                                "directBufferSize", "I");
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_init(
+                                JNIEnv *env, jclass cls, jint conserveMemory)
+{
+    bz_stream *stream = malloc(sizeof(bz_stream));
+    if (stream == 0) {
+        THROW(env, "java/lang/OutOfMemoryError", NULL);
+        return (jlong)0;
+    } 
+    memset((void*)stream, 0, sizeof(bz_stream));
+    
+    int rv = dlsym_BZ2_bzDecompressInit(stream, 0, conserveMemory);
+
+    if (rv != BZ_OK) {
+        // Contingency - Report error by throwing appropriate exceptions.
+        free(stream);
+        stream = NULL;
+
+        switch (rv) {
+        case BZ_MEM_ERROR:
+            {
+                THROW(env, "java/lang/OutOfMemoryError", NULL);
+            }
+            break;
+        default:
+            {
+                THROW(env, "java/lang/InternalError", NULL);
+            }
+            break;
+        }
+    }
+
+    return JLONG(stream);
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_inflateBytesDirect(
+                        JNIEnv *env, jobject this)
+{
+    // Get members of Bzip2Decompressor.
+    bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this,
+                                                Bzip2Decompressor_stream));
+    if (!stream) {
+        THROW(env, "java/lang/NullPointerException", NULL);
+        return (jint)0;
+    } 
+
+    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+                                                 Bzip2Decompressor_clazz);
+    jarray compressed_direct_buf = (jarray)(*env)->GetObjectField(env,
+                                this, Bzip2Decompressor_compressedDirectBuf);
+    jint compressed_direct_buf_off = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_compressedDirectBufOff);
+    jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_compressedDirectBufLen);
+
+    jarray uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env,
+                                this, Bzip2Decompressor_uncompressedDirectBuf);
+    jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_directBufferSize);
+
+    // Get the input and output direct buffers.
+    LOCK_CLASS(env, clazz, "Bzip2Decompressor");
+    char* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                compressed_direct_buf);
+    char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                uncompressed_direct_buf);
+    UNLOCK_CLASS(env, clazz, "Bzip2Decompressor");
+
+    if (!compressed_bytes || !uncompressed_bytes) {
+        return (jint)0;
+    }
+        
+    // Re-calibrate the bz_stream.
+    stream->next_in  = compressed_bytes + compressed_direct_buf_off;
+    stream->avail_in  = compressed_direct_buf_len;
+    stream->next_out = uncompressed_bytes;
+    stream->avail_out = uncompressed_direct_buf_len;
+        
+    // Decompress.
+    int rv = dlsym_BZ2_bzDecompress(stream);
+
+    // Contingency? - Report error by throwing appropriate exceptions.
+    int no_decompressed_bytes = 0;      
+    switch (rv) {
+    case BZ_STREAM_END:
+        {
+            (*env)->SetBooleanField(env, this,
+                                    Bzip2Decompressor_finished,
+                                    JNI_TRUE);
+        } // cascade down
+    case BZ_OK:
+        {
+            compressed_direct_buf_off +=
+                compressed_direct_buf_len - stream->avail_in;
+            (*env)->SetIntField(env, this,
+                                Bzip2Decompressor_compressedDirectBufOff, 
+                                compressed_direct_buf_off);
+            (*env)->SetIntField(env, this,
+                                Bzip2Decompressor_compressedDirectBufLen, 
+                                stream->avail_in);
+            no_decompressed_bytes =
+                uncompressed_direct_buf_len - stream->avail_out;
+        }
+        break;
+    case BZ_DATA_ERROR:
+    case BZ_DATA_ERROR_MAGIC:
+        {
+            THROW(env, "java/io/IOException", NULL);
+        }
+        break;
+    case BZ_MEM_ERROR:
+        {
+            THROW(env, "java/lang/OutOfMemoryError", NULL);
+        }
+        break;
+    default:
+        {
+            THROW(env, "java/lang/InternalError", NULL);
+        }
+        break;
+    }
+    
+    return no_decompressed_bytes;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesRead(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesWritten(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32;
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getRemaining(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    return (BZSTREAM(stream))->avail_in;
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_end(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    if (dlsym_BZ2_bzDecompressEnd(BZSTREAM(stream)) != BZ_OK) {
+        THROW(env, "java/lang/InternalError", 0);
+    } else {
+        free(BZSTREAM(stream));
+    }
+}
+
+/**
+ * vim: sw=2: ts=2: et:
+ */
+

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H
+#define ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H
+
+#include <config.h>
+#include <stddef.h>
+#include <bzlib.h>
+#include <dlfcn.h>
+#include <jni.h>
+
+#include "org_apache_hadoop.h"
+
+#define HADOOP_BZIP2_LIBRARY "libbz2.so.1"
+
+
+/* A helper macro to convert the java 'stream-handle' to a bz_stream pointer. */
+#define BZSTREAM(stream) ((bz_stream*)((ptrdiff_t)(stream)))
+
+/* A helper macro to convert the bz_stream pointer to the java 'stream-handle'. */
+#define JLONG(stream) ((jlong)((ptrdiff_t)(stream)))
+
+#endif //ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c

@@ -16,10 +16,14 @@
  * limitations under the License.
  */
 
-#include "config.h"
+
 #include "org_apache_hadoop.h"
 #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h"
 
+#ifdef UNIX
+#include "config.h"
+#endif // UNIX
+
 //****************************
 // Simple Functions
 //****************************
@@ -61,6 +65,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_init
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char* uncompressed_bytes;
+  char *compressed_bytes;
+
   // Get members of Lz4Compressor
   jobject clazz = (*env)->GetStaticObjectField(env, thisj, Lz4Compressor_clazz);
   jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf);
@@ -70,7 +77,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "Lz4Compressor");
-  const char* uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Compressor");
 
   if (uncompressed_bytes == 0) {
@@ -79,7 +86,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "Lz4Compressor");
-  char* compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Compressor");
 
   if (compressed_bytes == 0) {

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c

@@ -16,10 +16,13 @@
  * limitations under the License.
  */
 
-#include "config.h"
 #include "org_apache_hadoop.h"
 #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h"
 
+#ifdef UNIX
+#include "config.h"
+#endif // UNIX
+
 int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int isize, int maxOutputSize);
 
 /*
@@ -58,6 +61,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_in
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_decompressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char *compressed_bytes;
+  char *uncompressed_bytes;
+
   // Get members of Lz4Decompressor
   jobject clazz = (*env)->GetStaticObjectField(env,thisj, Lz4Decompressor_clazz);
   jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_compressedDirectBuf);
@@ -67,7 +73,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "Lz4Decompressor");
-  const char* compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
 
   if (compressed_bytes == 0) {
@@ -76,7 +82,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "Lz4Decompressor");
-  char* uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
 
   if (uncompressed_bytes == 0) {

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c

@@ -16,12 +16,18 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
+#endif // UNIX
+
 #include "org_apache_hadoop_io_compress_snappy.h"
 #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
 
@@ -81,7 +87,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (uncompressed_bytes == 0) {
-    return 0;
+    return (jint)0;
   }
 
   // Get the output direct buffer
@@ -90,7 +96,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (compressed_bytes == 0) {
-    return 0;
+    return (jint)0;
   }
 
   /* size_t should always be 4 bytes or larger. */
@@ -109,3 +115,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   (*env)->SetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen, 0);
   return (jint)buf_len;
 }
+
+#endif //define HADOOP_SNAPPY_LIBRARY

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c

@@ -16,12 +16,18 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
 #include "config.h"
+#include <dlfcn.h>
+#endif
+
 #include "org_apache_hadoop_io_compress_snappy.h"
 #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
 
@@ -103,3 +109,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres
 
   return (jint)uncompressed_direct_buf_len;
 }
+
+#endif //define HADOOP_SNAPPY_LIBRARY

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c

@@ -16,12 +16,15 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
+#endif
+
 #include "org_apache_hadoop_io_compress_zlib.h"
 #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h"
 
@@ -35,48 +38,124 @@ static jfieldID ZlibCompressor_directBufferSize;
 static jfieldID ZlibCompressor_finish;
 static jfieldID ZlibCompressor_finished;
 
+#ifdef UNIX
 static int (*dlsym_deflateInit2_)(z_streamp, int, int, int, int, int, const char *, int);
 static int (*dlsym_deflate)(z_streamp, int);
 static int (*dlsym_deflateSetDictionary)(z_streamp, const Bytef *, uInt);
 static int (*dlsym_deflateReset)(z_streamp);
 static int (*dlsym_deflateEnd)(z_streamp);
+#endif
+
+#ifdef WINDOWS
+#include <Strsafe.h>
+typedef int (__cdecl *__dlsym_deflateInit2_) (z_streamp, int, int, int, int, int, const char *, int);
+typedef int (__cdecl *__dlsym_deflate) (z_streamp, int);
+typedef int (__cdecl *__dlsym_deflateSetDictionary) (z_streamp, const Bytef *, uInt);
+typedef int (__cdecl *__dlsym_deflateReset) (z_streamp);
+typedef int (__cdecl *__dlsym_deflateEnd) (z_streamp);
+static __dlsym_deflateInit2_ dlsym_deflateInit2_;
+static __dlsym_deflate dlsym_deflate;
+static __dlsym_deflateSetDictionary dlsym_deflateSetDictionary;
+static __dlsym_deflateReset dlsym_deflateReset;
+static __dlsym_deflateEnd dlsym_deflateEnd;
+
+// Try to load zlib.dll from the dir where hadoop.dll is located.
+HANDLE LoadZlibTryHadoopNativeDir() {
+  HMODULE libz = NULL;
+  PCWSTR HADOOP_DLL = L"hadoop.dll";
+  size_t HADOOP_DLL_LEN = 10;
+  WCHAR path[MAX_PATH] = { 0 };
+  BOOL isPathValid = FALSE;
+
+  // Get hadoop.dll full path
+  HMODULE hModule = GetModuleHandle(HADOOP_DLL);
+  if (hModule != NULL) {
+    if (GetModuleFileName(hModule, path, MAX_PATH) > 0) {
+      size_t size = 0;
+      if (StringCchLength(path, MAX_PATH, &size) == S_OK) {
+
+        // Update path variable to have the full path to the zlib.dll
+        size = size - HADOOP_DLL_LEN;
+        if (size >= 0) {
+          path[size] = L'\0';
+          if (StringCchCat(path, MAX_PATH, HADOOP_ZLIB_LIBRARY) == S_OK) {
+            isPathValid = TRUE;
+          }
+        }
+      }
+    }
+  }
+
+  if (isPathValid) {
+    libz = LoadLibrary(path);
+  }
+
+  // fallback to system paths
+  if (!libz) {
+    libz = LoadLibrary(HADOOP_ZLIB_LIBRARY);
+  }
+
+  return libz;
+}
+#endif
 
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_initIDs(
 	JNIEnv *env, jclass class
 	) {
+#ifdef UNIX
 	// Load libz.so
 	void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
-	if (!libz) {
+  if (!libz) {
 		THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load libz.so");
 	  	return;
 	}
+#endif
+
+#ifdef WINDOWS
+  HMODULE libz = LoadZlibTryHadoopNativeDir();
 
+  if (!libz) {
+		THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zlib1.dll");
+    return;
+	}
+#endif
+
+#ifdef UNIX
 	// Locate the requisite symbols from libz.so
 	dlerror();                                 // Clear any existing error
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd");
+#endif
+
+#ifdef WINDOWS
+  LOAD_DYNAMIC_SYMBOL(__dlsym_deflateInit2_, dlsym_deflateInit2_, env, libz, "deflateInit2_");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflate, dlsym_deflate, env, libz, "deflate");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateSetDictionary, dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateReset, dlsym_deflateReset, env, libz, "deflateReset");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateEnd, dlsym_deflateEnd, env, libz, "deflateEnd");
+#endif
 
 	// Initialize the requisite fieldIds
-    ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+    ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz",
                                                       "Ljava/lang/Class;");
     ZlibCompressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
     ZlibCompressor_finish = (*env)->GetFieldID(env, class, "finish", "Z");
     ZlibCompressor_finished = (*env)->GetFieldID(env, class, "finished", "Z");
-    ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
-    									"uncompressedDirectBuf", 
+    ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class,
+        "uncompressedDirectBuf",
     									"Ljava/nio/Buffer;");
-    ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class,
     										"uncompressedDirectBufOff", "I");
-    ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class,
     										"uncompressedDirectBufLen", "I");
-    ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
-    									"compressedDirectBuf", 
+    ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class,
+                      "compressedDirectBuf",
     									"Ljava/nio/Buffer;");
-    ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class,
     										"directBufferSize", "I");
 }
 
@@ -84,7 +163,9 @@ JNIEXPORT jlong JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
 	JNIEnv *env, jclass class, jint level, jint strategy, jint windowBits
 	) {
-	// Create a z_stream
+    int rv = 0;
+    static const int memLevel = 8; 							// See zconf.h
+	  // Create a z_stream
     z_stream *stream = malloc(sizeof(z_stream));
     if (!stream) {
 		THROW(env, "java/lang/OutOfMemoryError", NULL);
@@ -93,17 +174,16 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
     memset((void*)stream, 0, sizeof(z_stream));
 
 	// Initialize stream
-	static const int memLevel = 8; 							// See zconf.h
-    int rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits,
+    rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits,
     			memLevel, strategy, ZLIB_VERSION, sizeof(z_stream));
-    			
+
     if (rv != Z_OK) {
 	    // Contingency - Report error by throwing appropriate exceptions
 	    free(stream);
 	    stream = NULL;
-	
+
 		switch (rv) {
-			case Z_MEM_ERROR: 
+			case Z_MEM_ERROR:
 			    {
 		    		THROW(env, "java/lang/OutOfMemoryError", NULL);
 			    }
@@ -120,27 +200,28 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
 		    break;
 	    }
 	}
-	
+
     return JLONG(stream);
 }
 
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_setDictionary(
-	JNIEnv *env, jclass class, jlong stream, 
+	JNIEnv *env, jclass class, jlong stream,
 	jarray b, jint off, jint len
 	) {
+    int rv = 0;
     Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0);
     if (!buf) {
         return;
     }
-    int rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len);
+    rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len);
     (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0);
-    
+
     if (rv != Z_OK) {
     	// Contingency - Report error by throwing appropriate exceptions
 	    switch (rv) {
 		    case Z_STREAM_ERROR:
-			{	
+			{
 		    	THROW(env, "java/lang/IllegalArgumentException", NULL);
 			}
 			break;
@@ -157,75 +238,85 @@ JNIEXPORT jint JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect(
 	JNIEnv *env, jobject this
 	) {
+    jobject clazz = NULL;
+    jobject uncompressed_direct_buf = NULL;
+    jint uncompressed_direct_buf_off = 0;
+    jint uncompressed_direct_buf_len = 0;
+    jobject compressed_direct_buf = NULL;
+    jint compressed_direct_buf_len = 0;
+    jboolean finish;
+    Bytef* uncompressed_bytes = NULL;
+    Bytef* compressed_bytes = NULL;
+    int rv = 0;
+    jint no_compressed_bytes = 0;
 	// Get members of ZlibCompressor
     z_stream *stream = ZSTREAM(
-    						(*env)->GetLongField(env, this, 
+                (*env)->GetLongField(env, this,
     									ZlibCompressor_stream)
     					);
     if (!stream) {
 		THROW(env, "java/lang/NullPointerException", NULL);
 		return (jint)0;
-    } 
+    }
 
     // Get members of ZlibCompressor
-    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+    clazz = (*env)->GetStaticObjectField(env, this,
                                                  ZlibCompressor_clazz);
-	jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, 
+	uncompressed_direct_buf = (*env)->GetObjectField(env, this,
 									ZlibCompressor_uncompressedDirectBuf);
-	jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, 
+	uncompressed_direct_buf_off = (*env)->GetIntField(env, this,
 									ZlibCompressor_uncompressedDirectBufOff);
-	jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	uncompressed_direct_buf_len = (*env)->GetIntField(env, this,
 									ZlibCompressor_uncompressedDirectBufLen);
 
-	jobject compressed_direct_buf = (*env)->GetObjectField(env, this, 
+	compressed_direct_buf = (*env)->GetObjectField(env, this,
 									ZlibCompressor_compressedDirectBuf);
-	jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	compressed_direct_buf_len = (*env)->GetIntField(env, this,
 									ZlibCompressor_directBufferSize);
 
-	jboolean finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish);
+	finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish);
 
     // Get the input direct buffer
     LOCK_CLASS(env, clazz, "ZlibCompressor");
-	Bytef* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+    uncompressed_bytes = (*env)->GetDirectBufferAddress(env,
 											uncompressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibCompressor");
-    
+
   	if (uncompressed_bytes == 0) {
     	return (jint)0;
 	}
-	
+
     // Get the output direct buffer
     LOCK_CLASS(env, clazz, "ZlibCompressor");
-	Bytef* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+    compressed_bytes = (*env)->GetDirectBufferAddress(env,
 										compressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibCompressor");
 
   	if (compressed_bytes == 0) {
 		return (jint)0;
 	}
-	
+
 	// Re-calibrate the z_stream
   	stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off;
   	stream->next_out = compressed_bytes;
   	stream->avail_in = uncompressed_direct_buf_len;
-	stream->avail_out = compressed_direct_buf_len;
-	
+    stream->avail_out = compressed_direct_buf_len;
+
 	// Compress
-	int rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
+	rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
 
-	jint no_compressed_bytes = 0;
 	switch (rv) {
     	// Contingency? - Report error by throwing appropriate exceptions
   		case Z_STREAM_END:
   		{
   			(*env)->SetBooleanField(env, this, ZlibCompressor_finished, JNI_TRUE);
   		} // cascade
-	  	case Z_OK: 
+      case Z_OK:
 	  	{
 	  		uncompressed_direct_buf_off += uncompressed_direct_buf_len - stream->avail_in;
-			(*env)->SetIntField(env, this, 
+			(*env)->SetIntField(env, this,
 						ZlibCompressor_uncompressedDirectBufOff, uncompressed_direct_buf_off);
-			(*env)->SetIntField(env, this, 
+			(*env)->SetIntField(env, this,
 						ZlibCompressor_uncompressedDirectBufLen, stream->avail_in);
 			no_compressed_bytes = compressed_direct_buf_len - stream->avail_out;
 	  	}
@@ -238,7 +329,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect(
 		}
 		break;
   	}
-  	
+
   	return no_compressed_bytes;
 }