Merge r1449958 through r1453659 from trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1453669 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -138,3 +138,70 @@ Create a local staging version of the website (in /tmp/hadoop-site)
   $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
 
 ----------------------------------------------------------------------------------
+
+Building on Windows
+
+----------------------------------------------------------------------------------
+Requirements:
+
+* Windows System
+* JDK 1.6
+* Maven 3.0
+* Findbugs 1.3.9 (if running findbugs)
+* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
+* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
+* Windows SDK or Visual Studio 2010 Professional
+* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+
+If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
+Do not use Visual Studio Express.  It does not support compiling for 64-bit,
+which is problematic if running a 64-bit system.  The Windows SDK is free to
+download here:
+
+http://www.microsoft.com/en-us/download/details.aspx?id=8279
+
+----------------------------------------------------------------------------------
+Building:
+
+Keep the source code tree in a short path to avoid running into problems related
+to Windows maximum path length limitation.  (For example, C:\hdc).
+
+Run builds from a Windows SDK Command Prompt.  (Start, All Programs,
+Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt.)
+
+JAVA_HOME must be set, and the path must not contain spaces.  If the full path
+would contain spaces, then use the Windows short path instead.
+
+You must set the Platform environment variable to either x64 or Win32 depending
+on whether you're running a 64-bit or 32-bit system.  Note that this is
+case-sensitive.  It must be "Platform", not "PLATFORM" or "platform".
+Environment variables on Windows are usually case-insensitive, but Maven treats
+them as case-sensitive.  Failure to set this environment variable correctly will
+cause msbuild to fail while building the native code in hadoop-common.
+
+set Platform=x64 (when building on a 64-bit system)
+set Platform=Win32 (when building on a 32-bit system)
+
+Several tests require that the user must have the Create Symbolic Links
+privilege.
+
+All Maven goals are the same as described above, with the addition of profile
+-Pnative-win to trigger building Windows native components.  The native
+components are required (not optional) on Windows.  For example:
+
+ * Run tests                 : mvn -Pnative-win test
+
+----------------------------------------------------------------------------------
+Building distributions:
+
+Create binary distribution with native code and with documentation:
+
+  $ mvn package -Pdist,native-win,docs -DskipTests -Dtar
+
+Create source distribution:
+
+  $ mvn package -Pnative-win,src -DskipTests
+
+Create source and binary distributions with native code and documentation:
+
+  $ mvn package -Pdist,native-win,docs,src -DskipTests -Dtar

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -26,6 +26,9 @@
       <outputDirectory>/bin</outputDirectory>
       <excludes>
         <exclude>*.sh</exclude>
+        <exclude>*-config.cmd</exclude>
+        <exclude>start-*.cmd</exclude>
+        <exclude>stop-*.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -38,6 +41,7 @@
       <outputDirectory>/libexec</outputDirectory>
       <includes>
         <include>*-config.sh</include>
+        <include>*-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -46,9 +50,13 @@
       <outputDirectory>/sbin</outputDirectory>
       <includes>
         <include>*.sh</include>
+        <include>*.cmd</include>
       </includes>
       <excludes>
         <exclude>hadoop-config.sh</exclude>
+        <exclude>hadoop.cmd</exclude>
+        <exclude>hdfs.cmd</exclude>
+        <exclude>hadoop-config.cmd</exclude>
       </excludes>
       <fileMode>0755</fileMode>
     </fileSet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -33,6 +33,7 @@
       <outputDirectory>bin</outputDirectory>
       <includes>
         <include>yarn</include>
+        <include>yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -41,6 +42,7 @@
       <outputDirectory>libexec</outputDirectory>
       <includes>
         <include>yarn-config.sh</include>
+        <include>yarn-config.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -52,6 +54,8 @@
         <include>yarn-daemons.sh</include>
         <include>start-yarn.sh</include>
         <include>stop-yarn.sh</include>
+        <include>start-yarn.cmd</include>
+        <include>stop-yarn.cmd</include>
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
@@ -121,7 +125,7 @@
       </includes>
       <binaries>
         <attachmentClassifier>tests</attachmentClassifier>
-        <outputDirectory>share/hadoop/${hadoop.component}</outputDirectory>
+        <outputDirectory>share/hadoop/${hadoop.component}/test</outputDirectory>
         <includeDependencies>false</includeDependencies>
         <unpack>false</unpack>
       </binaries>

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -20,6 +20,10 @@ Trunk (Unreleased)
     HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation
     of NetworkTopology.  (Junping Du via szetszwo)
 
+    HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows
+    Azure environments. (See breakdown of tasks below for subtasks and
+    contributors)
+
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
@@ -153,6 +157,9 @@ Trunk (Unreleased)
     HADOOP-9112. test-patch should -1 for @Tests without a timeout 
     (Surenkumar Nihalani via bobby)
 
+    HADOOP-9163 The rpc msg in ProtobufRpcEngine.proto should be moved out to
+    avoid an extra copy (Sanjay Radia)
+
   BUG FIXES
 
     HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
@@ -337,6 +344,118 @@ Trunk (Unreleased)
 
     HADOOP-9190. packaging docs is broken. (Andy Isaacson via atm)
 
+  BREAKDOWN OF HADOOP-8562 SUBTASKS
+
+    HADOOP-8924. Hadoop Common creating package-info.java must not depend on
+    sh. (Chris Nauroth via suresh)
+
+    HADOOP-8945. Merge winutils from branch-1-win to branch-trunk-win.
+    (Bikas Saha, Chuan Liu, Giridharan Kesavan, Ivan Mitic, and Steve Maine
+    ported by Chris Nauroth via suresh)
+
+    HADOOP-8946. winutils: compile codebase during Maven build on 
+    branch-trunk-win. (Chris Nauroth via suresh)
+
+    HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to 
+    branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, 
+    Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, 
+    Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh)
+
+    HADOOP-8954. "stat" executable not found on Windows. (Bikas Saha, Ivan Mitic
+    ported by Chris Narouth via suresh)
+
+    HADOOP-8959. TestUserGroupInformation fails on Windows due to "id" executable
+    not found. (Bikas Saha, Ivan Mitic, ported by Chris Narouth via suresh)
+
+    HADOOP-8955. "chmod" executable not found on Windows.
+    (Chris Nauroth via suresh)
+
+    HADOOP-8960. TestMetricsServlet fails on Windows. (Ivan Mitic via suresh)
+
+    HADOOP-8961. GenericOptionsParser URI parsing failure on Windows.
+    (Ivan Mitic via suresh)
+
+    HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via 
+    suresh)
+
+    HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple
+    test suites to fail. (Chris Nauroth via suresh)
+
+    HADOOP-8978. TestTrash fails on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-8979. TestHttpServer fails on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-8953. Shell PathData parsing failures on Windows. (Arpit Agarwal via
+    suresh)
+
+    HADOOP-8975. TestFileContextResolveAfs fails on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-8977. Multiple FsShell test failures on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, 
+    Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh)
+
+    HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via
+    suresh)
+
+    HADOOP-9011. saveVersion.py does not include branch in version annotation.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9110. winutils ls off-by-one error indexing MONTHS array can cause 
+    access violation. (Chris Nauroth via suresh)
+
+    HADOOP-9056. Build native library on Windows. (Chuan Liu, Arpit Agarwal via
+    suresh)
+
+    HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh)
+
+    HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, 
+    and Bikas Saha via suresh)
+
+    HADOOP-9146. Fix sticky bit regression on branch-trunk-win.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9266. Fix javac, findbugs, and release audit warnings on
+    branch-trunk-win. (Chris Nauroth via suresh)
+
+    HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via
+    szetszwo)
+
+    HADOOP-9271. Revert Python build scripts from branch-trunk-win.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9313. Remove spurious mkdir from hadoop-config.cmd.
+    (Ivan Mitic via suresh)
+
+    HADOOP-9309. Test failures on Windows due to UnsatisfiedLinkError
+    in NativeCodeLoader#buildSupportsSnappy. (Arpit Agarwal via suresh)
+
+    HADOOP-9347. Add instructions to BUILDING.txt describing how to
+    build on Windows. (Chris Nauroth via suresh)
+
+    HADOOP-9348. Address TODO in winutils to add more command line usage
+    and examples. (Chris Nauroth via suresh)
+
+    HADOOP-9354. Windows native project files missing license headers.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9356. Remove remaining references to cygwin/cygpath from scripts.
+    (Chris Nauroth via suresh)
+
+    HADOOP-9232. JniBasedUnixGroupsMappingWithFallback fails on Windows
+    with UnsatisfiedLinkError. (Ivan Mitic via suresh)
+
+    HADOOP-9368. Add timeouts to new tests in branch-trunk-win.
+    (Arpit Agarwal via suresh)
+
+    HADOOP-9373. Merge CHANGES.branch-trunk-win.txt to CHANGES.txt.
+    (suresh)
+
+    HADOOP-9372. Fix bad timeout annotations on tests.
+    (Arpit Agarwal via suresh)
+
 Release 2.0.4-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -358,6 +477,10 @@ Release 2.0.4-beta - UNRELEASED
     HADOOP-9279. Document the need to build hadoop-maven-plugins for
     eclipse and separate project builds. (Tsuyoshi Ozawa via suresh)
 
+    HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh)
+
+    HADOOP-9343. Allow additional exceptions through the RPC layer. (sseth)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -385,6 +508,25 @@ Release 2.0.4-beta - UNRELEASED
     HADOOP-8569. CMakeLists.txt: define _GNU_SOURCE and _LARGEFILE_SOURCE.
     (Colin Patrick McCabe via atm)
 
+    HADOOP-9323. Fix typos in API documentation. (suresh)
+
+    HADOOP-7487. DF should throw a more reasonable exception when mount cannot
+    be determined. (Andrew Wang via atm)
+
+    HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern.
+    (Arpit Gupta via suresh)
+
+    HADOOP-9342. Remove jline from distribution. (thw via tucu)
+
+    HADOOP-9230. TestUniformSizeInputFormat fails intermittently. 
+    (kkambatl via tucu)
+
+    HADOOP-9349. Confusing output when running hadoop version from one hadoop 
+    installation when HADOOP_HOME points to another. (sandyr via tucu)
+
+    HADOOP-9337. org.apache.hadoop.fs.DF.getMount() does not work on Mac OS.
+    (Ivan A. Veselovsky via atm)
+
 Release 2.0.3-alpha - 2013-02-06 
 
   INCOMPATIBLE CHANGES
@@ -645,15 +787,15 @@ Release 2.0.3-alpha - 2013-02-06
     HADOOP-9124. SortedMapWritable violates contract of Map interface for
     equals() and hashCode(). (Surenkumar Nihalani via tomwhite)
 
+    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
+    HarFileSystem. (Chris Nauroth via szetszwo)
+
     HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and
     the synchronization of limitDecimalTo2(double) can be avoided.  (szetszwo)
 
     HADOOP-9260. Hadoop version may be not correct when starting name node or
     data node. (Chris Nauroth via jlowe)
 
-    HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in
-    HarFileSystem. (Chris Nauroth via szetszwo)
-
     HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via
     suresh)
 
@@ -1360,10 +1502,21 @@ Release 0.23.7 - UNRELEASED
     permissions (Ivan A. Veselovsky via bobby)
 
     HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure
-    (Ivan A. Veselovsky via bobby) 
+    (Ivan A. Veselovsky via bobby)
+
+    HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp
+    via kihwal)
+
+    HADOOP-9352. Expose UGI.setLoginUser for tests (daryn)
+
+    HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via
+    jeagles)
 
   OPTIMIZATIONS
 
+    HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via
+    jlowe)
+
   BUG FIXES
 
     HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via
@@ -1372,6 +1525,9 @@ Release 0.23.7 - UNRELEASED
     HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage
     option (Andy Isaacson via tgraves)
 
+    HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp via 
+    kihwal)
+
 Release 0.23.6 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/pom.xml

@@ -219,6 +219,10 @@
       <artifactId>zookeeper</artifactId>
       <version>3.4.2</version>
       <exclusions>
+        <exclusion>
+          <groupId>jline</groupId>
+          <artifactId>jline</artifactId>
+        </exclusion>
         <exclusion>
           <!-- otherwise seems to drag in junit 3.8.1 via jline -->
           <groupId>junit</groupId>
@@ -460,6 +464,7 @@
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
+        <require.bzip2>false</require.bzip2>
         <snappy.prefix></snappy.prefix>
         <snappy.lib></snappy.lib>
         <snappy.include></snappy.include>
@@ -467,6 +472,28 @@
       </properties>
       <build>
         <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>mac</family>
+                      <family>unix</family>
+                      <message>native build only supported on Mac or Unix</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>native-maven-plugin</artifactId>
@@ -481,6 +508,8 @@
                   <javahClassNames>
                     <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Compressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
                     <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
                     <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
@@ -506,7 +535,7 @@
                 <configuration>
                   <target>
                     <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                       <arg line="VERBOSE=1"/>
@@ -537,6 +566,104 @@
       </build>
     </profile>
 
+    <profile>
+      <id>native-win</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-os</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireOS>
+                      <family>windows</family>
+                      <message>native-win build only supported on Windows</message>
+                    </requireOS>
+                  </rules>  
+                  <fail>true</fail>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>native-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>javah</goal>
+                </goals>
+                <configuration>
+                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+                  <javahClassNames>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.zlib.ZlibDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.nativeio.NativeIO</javahClassName>
+                    <javahClassName>org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyCompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
+                  </javahClassNames>
+                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>compile-ms-winutils</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/winutils/winutils.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+              <execution>
+                <id>compile-ms-native-dll</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/native/native.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!-- profile that starts ApacheDS KDC server -->
     <profile>
       <id>startKdc</id>

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -97,6 +97,23 @@ set(T main/native/src/test/org/apache/hadoop)
 
 GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
 
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1")
+find_package(BZip2 QUIET)
+if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+    GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
+    set(BZIP2_SOURCE_FILES
+          "${D}/io/compress/bzip2/Bzip2Compressor.c"
+          "${D}/io/compress/bzip2/Bzip2Decompressor.c")
+else (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+    set(BZIP2_SOURCE_FILES "")
+    set(BZIP2_INCLUDE_DIR "")
+    IF(REQUIRE_BZIP2)
+        MESSAGE(FATAL_ERROR "Required bzip2 library and/or header files could not be found.")
+    ENDIF(REQUIRE_BZIP2)
+endif (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+
 INCLUDE(CheckFunctionExists)
 INCLUDE(CheckCSourceCompiles)
 INCLUDE(CheckLibraryExists)
@@ -136,6 +153,7 @@ include_directories(
     ${CMAKE_BINARY_DIR}
     ${JNI_INCLUDE_DIRS}
     ${ZLIB_INCLUDE_DIRS}
+    ${BZIP2_INCLUDE_DIR}
     ${SNAPPY_INCLUDE_DIR}
     ${D}/util
 )
@@ -155,6 +173,7 @@ add_dual_library(hadoop
     ${SNAPPY_SOURCE_FILES}
     ${D}/io/compress/zlib/ZlibCompressor.c
     ${D}/io/compress/zlib/ZlibDecompressor.c
+    ${BZIP2_SOURCE_FILES}
     ${D}/io/nativeio/NativeIO.c
     ${D}/io/nativeio/errno_enum.c
     ${D}/io/nativeio/file_descriptor.c

hadoop-common-project/hadoop-common/src/config.h.cmake

@@ -19,6 +19,7 @@
 #define CONFIG_H
 
 #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
+#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
 #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
 #cmakedefine HAVE_SYNC_FILE_RANGE
 #cmakedefine HAVE_POSIX_FADVISE

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -91,9 +91,6 @@ case $COMMAND in
     ;;
 
   classpath)
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     echo $CLASSPATH
     exit
     ;;
@@ -132,9 +129,6 @@ case $COMMAND in
     #make sure security appender is turned off
     HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
 
-    if $cygwin; then
-      CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-    fi
     export CLASSPATH=$CLASSPATH
     exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
     ;;

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd

@@ -0,0 +1,292 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem included in all the hadoop scripts with source command
+@rem should not be executable directly
+@rem also should not be passed any arguments, since we need original %*
+
+if not defined HADOOP_COMMON_DIR (
+  set HADOOP_COMMON_DIR=share\hadoop\common
+)
+if not defined HADOOP_COMMON_LIB_JARS_DIR (
+  set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib
+)
+if not defined HADOOP_COMMON_LIB_NATIVE_DIR (
+  set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native
+)
+if not defined HDFS_DIR (
+  set HDFS_DIR=share\hadoop\hdfs
+)
+if not defined HDFS_LIB_JARS_DIR (
+  set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib
+)
+if not defined YARN_DIR (
+  set YARN_DIR=share\hadoop\yarn
+)
+if not defined YARN_LIB_JARS_DIR (
+  set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib
+)
+if not defined MAPRED_DIR (
+  set MAPRED_DIR=share\hadoop\mapreduce
+)
+if not defined MAPRED_LIB_JARS_DIR (
+  set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib
+)
+
+@rem the root of the Hadoop installation
+set HADOOP_HOME=%~dp0
+for %%i in (%HADOOP_HOME%.) do (
+  set HADOOP_HOME=%%~dpi
+)
+if "%HADOOP_HOME:~-1%" == "\" (
+  set HADOOP_HOME=%HADOOP_HOME:~0,-1%
+)
+
+if not exist %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar (
+    @echo +================================================================+
+    @echo ^|      Error: HADOOP_HOME is not set correctly                   ^|
+    @echo +----------------------------------------------------------------+
+    @echo ^| Please set your HADOOP_HOME variable to the absolute path of   ^|
+    @echo ^| the directory that contains the hadoop distribution            ^|
+    @echo +================================================================+
+    exit /b 1
+)
+
+set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
+
+@rem
+@rem Allow alternate conf dir location.
+@rem
+
+if "%1" == "--config" (
+  set HADOOP_CONF_DIR=%2
+  shift
+  shift
+)
+
+@rem
+@rem check to see it is specified whether to use the slaves or the
+@rem masters file
+@rem
+
+if "%1" == "--hosts" (
+  set HADOOP_SLAVES=%HADOOP_CONF_DIR%\%2
+  shift
+  shift
+)
+
+if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
+  call %HADOOP_CONF_DIR%\hadoop-env.cmd
+)
+
+@rem
+@rem setup java environment variables
+@rem
+
+if not defined JAVA_HOME (
+  echo Error: JAVA_HOME is not set.
+  goto :eof
+)
+
+if not exist %JAVA_HOME%\bin\java.exe (
+  echo Error: JAVA_HOME is incorrectly set.
+  echo        Please update %HADOOP_HOME%\conf\hadoop-env.cmd
+  goto :eof
+)
+
+set JAVA=%JAVA_HOME%\bin\java
+@rem some Java parameters
+set JAVA_HEAP_MAX=-Xmx1000m
+
+@rem
+@rem check envvars which might override default args
+@rem
+
+if defined HADOOP_HEAPSIZE (
+  set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m
+)
+
+@rem
+@rem CLASSPATH initially contains %HADOOP_CONF_DIR%
+@rem
+
+set CLASSPATH=%HADOOP_CONF_DIR%
+
+if not defined HADOOP_COMMON_HOME (
+  if exist %HADOOP_HOME%\share\hadoop\common (
+    set HADOOP_COMMON_HOME=%HADOOP_HOME%
+  )
+)
+
+@rem
+@rem for releases, add core hadoop jar & webapps to CLASSPATH
+@rem
+
+if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%
+)
+
+if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\*
+
+@rem
+@rem add user-specified CLASSPATH last
+@rem
+
+if defined HADOOP_CLASSPATH (
+  if defined HADOOP_USER_CLASSPATH_FIRST (
+    set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
+  ) else (
+    set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
+  )
+)
+
+@rem
+@rem default log directory % file
+@rem
+
+if not defined HADOOP_LOG_DIR (
+  set HADOOP_LOG_DIR=%HADOOP_HOME%\logs
+)
+
+if not defined HADOOP_LOGFILE (
+  set HADOOP_LOGFILE=hadoop.log
+)
+
+if not defined HADOOP_ROOT_LOGGER (
+  set HADOOP_ROOT_LOGGER=INFO,console
+)
+
+@rem
+@rem default policy file for service-level authorization
+@rem
+
+if not defined HADOOP_POLICYFILE (
+  set HADOOP_POLICYFILE=hadoop-policy.xml
+)
+
+@rem
+@rem Determine the JAVA_PLATFORM
+@rem
+
+for /f "delims=" %%A in ('%JAVA% -Xmx32m %HADOOP_JAVA_PLATFORM_OPTS% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName') do set JAVA_PLATFORM=%%A
+@rem replace space with underscore
+set JAVA_PLATFORM=%JAVA_PLATFORM: =_%
+
+@rem
+@rem setup 'java.library.path' for native hadoop code if necessary
+@rem
+
+@rem Check if we're running hadoop directly from the build
+set JAVA_LIBRARY_PATH=
+if exist %HADOOP_CORE_HOME%\target\bin (
+  set JAVA_LIBRARY_PATH=%HADOOP_CORE_HOME%\target\bin
+)
+
+@rem For the distro case, check the bin folder
+if exist %HADOOP_CORE_HOME%\bin (
+  set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\bin
+)
+
+@rem
+@rem setup a default TOOL_PATH
+@rem
+set TOOL_PATH=%HADOOP_HOME%\share\hadoop\tools\lib\*
+
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_HOME%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING%
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER%
+
+if defined JAVA_LIBRARY_PATH (
+  set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE%
+
+@rem
+@rem Disable ipv6 as it can cause issues
+@rem
+
+set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
+
+@rem
+@rem put hdfs in classpath if present
+@rem
+
+if not defined HADOOP_HDFS_HOME (
+  if exist %HADOOP_HOME%\%HDFS_DIR% (
+    set HADOOP_HDFS_HOME=%HADOOP_HOME%
+  )
+)
+
+if exist %HADOOP_HDFS_HOME%\%HDFS_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%
+)
+
+if exist %HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\*
+
+@rem
+@rem put yarn in classpath if present
+@rem
+
+if not defined HADOOP_YARN_HOME (
+  if exist %HADOOP_HOME%\%YARN_DIR% (
+    set HADOOP_YARN_HOME=%HADOOP_HOME%
+  )
+)
+
+if exist %HADOOP_YARN_HOME%\%YARN_DIR%\webapps (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%
+)
+
+if exist %HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR% (
+  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
+)
+
+set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\*
+
+@rem
+@rem put mapred in classpath if present AND different from YARN
+@rem
+
+if not defined HADOOP_MAPRED_HOME (
+  if exist %HADOOP_HOME%\%MAPRED_DIR% (
+    set HADOOP_MAPRED_HOME=%HADOOP_HOME%
+  )
+)
+
+if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" (
+
+  if exist %HADOOP_MAPRED_HOME%\%MAPRED_DIR%\webapps (
+    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%
+  )
+
+  if exist %HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR% (
+    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\*
+  )
+
+  set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
+)
+
+:eof

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh

@@ -112,12 +112,6 @@ if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then
   exit 1
 fi
 
-cygwin=false
-case "`uname`" in
-CYGWIN*) cygwin=true;;
-esac
-
-
 # check if net.ipv6.bindv6only is set to 1
 bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
 if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] && [ "$HADOOP_ALLOW_IPV6" != "yes" ]
@@ -209,13 +203,6 @@ fi
 # restore ordinary behaviour
 unset IFS
 
-# cygwin path translation
-if $cygwin; then
-  HADOOP_PREFIX=`cygpath -w "$HADOOP_PREFIX"`
-  HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"`
-  JAVA_LIBRARY_PATH=`cygpath -w "$JAVA_LIBRARY_PATH"`
-fi
-
 # setup 'java.library.path' for native-hadoop code if necessary
 
 if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then
@@ -232,11 +219,6 @@ fi
 # setup a default TOOL_PATH
 TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
 
-# cygwin path translation
-if $cygwin; then
-  JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
-fi
-
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
@@ -303,15 +285,3 @@ if [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" != "$HADOOP_YARN_HOME/$YARN_DIR" ] ; then
 
   CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR'/*'
 fi
-
-# cygwin path translation
-if $cygwin; then
-  HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`
-fi
-
-# cygwin path translation
-if $cygwin; then
-  TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
-fi
-
-

hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd

@@ -0,0 +1,235 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+
+@rem This script runs the hadoop core commands. 
+
+@rem Environment Variables
+@rem
+@rem   JAVA_HOME        The java implementation to use.  Overrides JAVA_HOME.
+@rem
+@rem   HADOOP_CLASSPATH Extra Java CLASSPATH entries.
+@rem
+@rem   HADOOP_USER_CLASSPATH_FIRST      When defined, the HADOOP_CLASSPATH is
+@rem                                    added in the beginning of the global
+@rem                                    classpath. Can be defined, for example,
+@rem                                    by doing
+@rem                                    export HADOOP_USER_CLASSPATH_FIRST=true
+@rem
+@rem   HADOOP_HEAPSIZE  The maximum amount of heap to use, in MB.
+@rem                    Default is 1000.
+@rem
+@rem   HADOOP_OPTS      Extra Java runtime options.
+@rem
+@rem   HADOOP_CLIENT_OPTS         when the respective command is run.
+@rem   HADOOP_{COMMAND}_OPTS etc  HADOOP_JT_OPTS applies to JobTracker
+@rem                              for e.g.  HADOOP_CLIENT_OPTS applies to
+@rem                              more than one command (fs, dfs, fsck,
+@rem                              dfsadmin etc)
+@rem
+@rem   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
+@rem
+@rem   HADOOP_ROOT_LOGGER The root appender. Default is INFO,console
+@rem
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+call :updatepath %HADOOP_BIN_PATH%
+
+:main
+  setlocal enabledelayedexpansion
+
+  set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+  if not defined HADOOP_LIBEXEC_DIR (
+    set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+  )
+
+  call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+  if "%1" == "--config" (
+    shift
+    shift
+  )
+
+  set hadoop-command=%1
+  if not defined hadoop-command (
+      goto print_usage
+  )
+
+  call :make_command_arguments %*
+
+  set hdfscommands=namenode secondarynamenode datanode dfs dfsadmin fsck balancer fetchdt oiv dfsgroups
+  for %%i in ( %hdfscommands% ) do (
+    if %hadoop-command% == %%i set hdfscommand=true
+  )
+  if defined hdfscommand (
+    @echo DEPRECATED: Use of this script to execute hdfs command is deprecated. 1>&2
+    @echo Instead use the hdfs command for it. 1>&2
+    if exist %HADOOP_HDFS_HOME%\bin\hdfs.cmd (
+      call %HADOOP_HDFS_HOME%\bin\hdfs.cmd %*
+      goto :eof
+    ) else if exist %HADOOP_HOME%\bin\hdfs.cmd (
+      call %HADOOP_HOME%\bin\hdfs.cmd %*
+      goto :eof
+    ) else (
+      echo HADOOP_HDFS_HOME not found!
+      goto :eof
+    )
+  )
+
+  set mapredcommands=pipes job queue mrgroups mradmin jobtracker tasktracker
+  for %%i in ( %mapredcommands% ) do (
+    if %hadoop-command% == %%i set mapredcommand=true  
+  )
+  if defined mapredcommand (
+    @echo DEPRECATED: Use of this script to execute mapred command is deprecated. 1>&2
+    @echo Instead use the mapred command for it. 1>&2
+    if exist %HADOOP_MAPRED_HOME%\bin\mapred.cmd (
+      call %HADOOP_MAPRED_HOME%\bin\mapred.cmd %*
+      goto :eof
+    ) else if exist %HADOOP_HOME%\bin\mapred.cmd (
+      call %HADOOP_HOME%\bin\mapred.cmd %*
+      goto :eof
+    ) else (
+      echo HADOOP_MAPRED_HOME not found!
+      goto :eof
+    )
+  )
+
+  if %hadoop-command% == classpath (
+    @echo %CLASSPATH%
+    goto :eof
+  )
+  
+  set corecommands=fs version jar distcp daemonlog archive
+  for %%i in ( %corecommands% ) do (
+    if %hadoop-command% == %%i set corecommand=true  
+  )
+  if defined corecommand (
+    call :%hadoop-command%
+  ) else (
+    set CLASSPATH=%CLASSPATH%;%CD%
+    set CLASS=%hadoop-command%
+  )
+
+  set path=%PATH%;%HADOOP_BIN_PATH%
+
+  @rem Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
+  set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS%
+
+  @rem make sure security appender is turned off
+  if not defined HADOOP_SECURITY_LOGGER (
+    set HADOOP_SECURITY_LOGGER=INFO,NullAppender
+  )
+  set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER%
+
+  call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hadoop-command-arguments%
+
+  goto :eof
+
+:fs 
+  set CLASS=org.apache.hadoop.fs.FsShell
+  goto :eof
+
+:version 
+  set CLASS=org.apache.hadoop.util.VersionInfo
+  goto :eof
+
+:jar
+  set CLASS=org.apache.hadoop.util.RunJar
+  goto :eof
+
+:distcp
+  set CLASS=org.apache.hadoop.tools.DistCp
+  set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
+  goto :eof
+
+:daemonlog
+  set CLASS=org.apache.hadoop.log.LogLevel
+  goto :eof
+
+:archive
+  set CLASS=org.apache.hadoop.tools.HadoopArchives
+  set CLASSPATH=%CLASSPATH%;%TOOL_PATH%
+  goto :eof
+
+:updatepath
+  set path_to_add=%*
+  set current_path_comparable=%path%
+  set current_path_comparable=%current_path_comparable: =_%
+  set current_path_comparable=%current_path_comparable:(=_%
+  set current_path_comparable=%current_path_comparable:)=_%
+  set path_to_add_comparable=%path_to_add%
+  set path_to_add_comparable=%path_to_add_comparable: =_%
+  set path_to_add_comparable=%path_to_add_comparable:(=_%
+  set path_to_add_comparable=%path_to_add_comparable:)=_%
+
+  for %%i in ( %current_path_comparable% ) do (
+    if /i "%%i" == "%path_to_add_comparable%" (
+      set path_to_add_exist=true
+    )
+  )
+  set system_path_comparable=
+  set path_to_add_comparable=
+  if not defined path_to_add_exist path=%path_to_add%;%path%
+  set path_to_add=
+  goto :eof
+
+@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
+:make_command_arguments
+  if "%1" == "--config" (
+    shift
+    shift
+  )
+  if [%2] == [] goto :eof
+  shift
+  set _arguments=
+  :MakeCmdArgsLoop 
+  if [%1]==[] goto :EndLoop 
+
+  if not defined _arguments (
+    set _arguments=%1
+  ) else (
+    set _arguments=!_arguments! %1
+  )
+  shift
+  goto :MakeCmdArgsLoop 
+  :EndLoop 
+  set hadoop-command-arguments=%_arguments%
+  goto :eof
+
+:print_usage
+  @echo Usage: hadoop [--config confdir] COMMAND
+  @echo where COMMAND is one of:
+  @echo   fs                   run a generic filesystem user client
+  @echo   version              print the version
+  @echo   jar ^<jar^>            run a jar file
+  @echo   distcp ^<srcurl^> ^<desturl^> copy file or directories recursively
+  @echo   archive -archiveName NAME -p ^<parent path^> ^<src^>* ^<dest^> create a hadoop archive
+  @echo   classpath            prints the class path needed to get the
+  @echo                        Hadoop jar and the required libraries
+  @echo   daemonlog            get/set the log level for each daemon
+  @echo  or
+  @echo   CLASSNAME            run the class named CLASSNAME
+  @echo.
+  @echo Most commands print help when invoked w/o parameters.
+
+endlocal

hadoop-common-project/hadoop-common/src/main/bin/rcc

@@ -57,10 +57,5 @@ unset IFS
 
 CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
 
-# cygwin path translation
-if expr `uname` : 'CYGWIN*' > /dev/null; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
 # run it
 exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@"

hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd

@@ -0,0 +1,52 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+setlocal enabledelayedexpansion
+
+@rem Start all hadoop daemons.  Run this on master node.
+
+echo This script is Deprecated. Instead use start-dfs.cmd and start-yarn.cmd
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+  set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+if "%1" == "--config" (
+  shift
+  shift
+)
+
+@rem start hdfs daemons if hdfs is present
+if exist %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd (
+  call %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR%
+)
+
+@rem start yarn daemons if yarn is present
+if exist %HADOOP_YARN_HOME%\sbin\start-yarn.cmd (
+  call %HADOOP_YARN_HOME%\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR%
+)
+
+endlocal

hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd

@@ -0,0 +1,52 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+setlocal enabledelayedexpansion
+
+@rem Stop all hadoop daemons.  Run this on master node.
+
+echo This script is Deprecated. Instead use stop-dfs.cmd and stop-yarn.cmd
+
+if not defined HADOOP_BIN_PATH ( 
+  set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+  set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+  set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+if "%1" == "--config" (
+  shift
+  shift
+)
+
+@rem stop hdfs daemons if hdfs is present
+if exist %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd (
+  call %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd --config %HADOOP_CONF_DIR%
+)
+
+@rem stop yarn daemons if yarn is present
+if exist %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd (
+  call %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd --config %HADOOP_CONF_DIR%
+)
+
+endlocal

hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd

@@ -0,0 +1,81 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements.  See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License.  You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Set Hadoop-specific environment variables here.
+
+@rem The only required environment variable is JAVA_HOME.  All others are
+@rem optional.  When running a distributed configuration it is best to
+@rem set JAVA_HOME in this file, so that it is correctly defined on
+@rem remote nodes.
+
+@rem The java implementation to use.  Required.
+set JAVA_HOME=%JAVA_HOME%
+
+@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
+@rem set JSVC_HOME=%JSVC_HOME%
+
+@rem set HADOOP_CONF_DIR=
+
+@rem Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
+if exist %HADOOP_HOME%\contrib\capacity-scheduler (
+  if not defined HADOOP_CLASSPATH (
+    set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+  ) else (
+    set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
+  )
+)
+
+@rem The maximum amount of heap to use, in MB. Default is 1000.
+@rem set HADOOP_HEAPSIZE=
+@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+@rem Extra Java runtime options.  Empty by default.
+@rem set HADOOP_OPTS=-Djava.net.preferIPv4Stack=true %HADOOP_CLIENT_OPTS%
+
+@rem Command specific options appended to HADOOP_OPTS when specified
+if not defined HADOOP_SECURITY_LOGGER (
+  set HADOOP_SECURITY_LOGGER=INFO,RFAS
+)
+if not defined HDFS_AUDIT_LOGGER (
+  set HDFS_AUDIT_LOGGER=INFO,NullAppender
+)
+
+set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
+set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
+set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
+
+@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS%
+@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
+
+@rem On secure datanodes, user to run the datanode as after dropping privileges
+set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
+
+@rem Where log files are stored.  %HADOOP_HOME%/logs by default.
+@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
+
+@rem Where log files are stored in the secure data environment.
+set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
+
+@rem The directory where pid files are stored. /tmp by default.
+@rem NOTE: this should be set to a directory that can only be written to by 
+@rem       the user that will run the hadoop daemons.  Otherwise there is the
+@rem       potential for a symlink attack.
+set HADOOP_PID_DIR=%HADOOP_PID_DIR%
+set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
+
+@rem A string representing this instance of hadoop. %USERNAME% by default.
+set HADOOP_IDENT_STRING=%USERNAME%

hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml

@@ -87,7 +87,6 @@ See http://forrest.apache.org/docs/linking.html for more info.
     <zlib      href="http://www.zlib.net/" />
     <gzip      href="http://www.gzip.org/" />
     <bzip      href="http://www.bzip.org/" />
-    <cygwin    href="http://www.cygwin.com/" />
     <osx       href="http://www.apple.com/macosx" />
     
     <relnotes href="releasenotes.html" />

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java

@@ -21,8 +21,6 @@ package org.apache.hadoop.fs;
 import java.io.*;
 import java.util.Arrays;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -32,7 +30,7 @@ import org.apache.hadoop.util.PureJavaCrc32;
 
 /****************************************************************
  * Abstract Checksumed FileSystem.
- * It provide a basice implementation of a Checksumed FileSystem,
+ * It provide a basic implementation of a Checksumed FileSystem,
  * which creates a checksum file for each raw file.
  * It generates & verifies checksums at the client side.
  *
@@ -118,9 +116,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
    * It verifies that data matches checksums.
    *******************************************************/
   private static class ChecksumFSInputChecker extends FSInputChecker {
-    public static final Log LOG 
-      = LogFactory.getLog(FSInputChecker.class);
-    
     private ChecksumFileSystem fs;
     private FSDataInputStream datas;
     private FSDataInputStream sums;
@@ -374,19 +369,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
     private FSDataOutputStream sums;
     private static final float CHKSUM_AS_FRACTION = 0.01f;
     
-    public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
-                          Path file, 
-                          boolean overwrite, 
-                          short replication,
-                          long blockSize,
-                          Configuration conf)
-      throws IOException {
-      this(fs, file, overwrite, 
-           conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
-		       LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT),
-           replication, blockSize, null);
-    }
-    
     public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
                           Path file, 
                           boolean overwrite,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java

@@ -17,22 +17,25 @@
  */
 package org.apache.hadoop.fs;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.BufferedReader;
-
+import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.NoSuchElementException;
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.util.Shell;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /** Filesystem disk space usage statistics.
  * Uses the unix 'df' program to get mount points, and java.io.File for
- * space utilization. Tested on Linux, FreeBSD, Cygwin. */
+ * space utilization. Tested on Linux, FreeBSD, Windows. */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public class DF extends Shell {
@@ -44,6 +47,8 @@ public class DF extends Shell {
   private final File dirFile;
   private String filesystem;
   private String mount;
+  
+  private ArrayList<String> output;
 
   enum OSType {
     OS_TYPE_UNIX("UNIX"),
@@ -84,6 +89,7 @@ public class DF extends Shell {
     super(dfInterval);
     this.dirPath = path.getCanonicalPath();
     this.dirFile = new File(this.dirPath);
+    this.output = new ArrayList<String>();
   }
 
   protected OSType getOSType() {
@@ -127,7 +133,21 @@ public class DF extends Shell {
 
   /** @return the filesystem mount point for the indicated volume */
   public String getMount() throws IOException {
+    // Abort early if specified path does not exist
+    if (!dirFile.exists()) {
+      throw new FileNotFoundException("Specified path " + dirFile.getPath()
+          + "does not exist");
+    }
     run();
+    // Skip parsing if df was not successful
+    if (getExitCode() != 0) {
+      StringBuffer sb = new StringBuffer("df could not be run successfully: ");
+      for (String line: output) {
+        sb.append(line);
+      }
+      throw new IOException(sb.toString());
+    }
+    parseOutput();
     return mount;
   }
   
@@ -143,55 +163,76 @@ public class DF extends Shell {
       mount;
   }
 
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      try {
+        this.mount = dirFile.getCanonicalPath().substring(0,2);
+      } catch (IOException e) {
+      }
+      return;
+    }
+    super.run();
+  }
+
   @Override
   protected String[] getExecString() {
     // ignoring the error since the exit code it enough
-    return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath 
-                         + "' 2>/dev/null"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "df -k " + dirPath + " 2>nul"}:
+        new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath 
+                      + "' 2>/dev/null"};
   }
 
   @Override
   protected void parseExecResult(BufferedReader lines) throws IOException {
-    lines.readLine();                         // skip headings
-  
+    output.clear();
     String line = lines.readLine();
-    if (line == null) {
-      throw new IOException( "Expecting a line not the end of stream" );
+    while (line != null) {
+      output.add(line);
+      line = lines.readLine();
     }
+  }
+  
+  @VisibleForTesting
+  protected void parseOutput() throws IOException {
+    if (output.size() < 2) {
+      StringBuffer sb = new StringBuffer("Fewer lines of output than expected");
+      if (output.size() > 0) {
+        sb.append(": " + output.get(0));
+      }
+      throw new IOException(sb.toString());
+    }
+    
+    String line = output.get(1);
     StringTokenizer tokens =
       new StringTokenizer(line, " \t\n\r\f%");
     
-    this.filesystem = tokens.nextToken();
+    try {
+      this.filesystem = tokens.nextToken();
+    } catch (NoSuchElementException e) {
+      throw new IOException("Unexpected empty line");
+    }
     if (!tokens.hasMoreTokens()) {            // for long filesystem name
-      line = lines.readLine();
-      if (line == null) {
-        throw new IOException( "Expecting a line not the end of stream" );
+      if (output.size() > 2) {
+        line = output.get(2);
+      } else {
+        throw new IOException("Expecting additional output after line: "
+            + line);
       }
       tokens = new StringTokenizer(line, " \t\n\r\f%");
     }
 
-    switch(getOSType()) {
-      case OS_TYPE_AIX:
-        Long.parseLong(tokens.nextToken()); // capacity
-        Long.parseLong(tokens.nextToken()); // available
-        Integer.parseInt(tokens.nextToken()); // pct used
-        tokens.nextToken();
-        tokens.nextToken();
-        this.mount = tokens.nextToken();
-        break;
-
-      case OS_TYPE_WIN:
-      case OS_TYPE_SOLARIS:
-      case OS_TYPE_MAC:
-      case OS_TYPE_UNIX:
-      default:
-        Long.parseLong(tokens.nextToken()); // capacity
-        Long.parseLong(tokens.nextToken()); // used
-        Long.parseLong(tokens.nextToken()); // available
-        Integer.parseInt(tokens.nextToken()); // pct used
-        this.mount = tokens.nextToken();
-        break;
-   }
+    try {
+      Long.parseLong(tokens.nextToken()); // capacity
+      Long.parseLong(tokens.nextToken()); // used
+      Long.parseLong(tokens.nextToken()); // available
+      Integer.parseInt(tokens.nextToken()); // pct used
+      this.mount = tokens.nextToken();
+    } catch (NoSuchElementException e) {
+      throw new IOException("Could not parse line: " + line);
+    } catch (NumberFormatException e) {
+      throw new IOException("Could not parse line: " + line);
+    }
   }
 
   public static void main(String[] args) throws Exception {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java

@@ -145,6 +145,20 @@ public class DU extends Shell {
   public String getDirPath() {
     return dirPath;
   }
+
+
+  /**
+   * Override to hook in DUHelper class. Maybe this can be used more
+   * generally as well on Unix/Linux based systems
+   */
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      used.set(DUHelper.getFolderUsage(dirPath));
+      return;
+    }
+    super.run();
+  }
   
   /**
    * Start the disk usage checking thread.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java

@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.File;
+import org.apache.hadoop.util.Shell;
+
+public class DUHelper {
+
+  private int folderCount=0;
+  private int fileCount=0;
+  private double usage = 0;
+  private long folderSize = -1;
+
+  private DUHelper() {
+
+  }
+
+  public static long getFolderUsage(String folder) {
+    return new DUHelper().calculateFolderSize(folder);
+  }
+
+  private long calculateFolderSize(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+    return folderSize = getFileSize(f);
+  }
+
+  public String check(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+
+    folderSize = getFileSize(f);
+    usage = 1.0*(f.getTotalSpace() - f.getFreeSpace())/ f.getTotalSpace();
+    return String.format("used %d files %d disk in use %f", folderSize, fileCount, usage);
+  }
+
+  public long getFileCount() {
+    return fileCount;
+  }
+
+  public double getUsage() {
+    return usage;
+  }
+
+  private long getFileSize(File folder) {
+
+    folderCount++;
+    //Counting the total folders
+    long foldersize = 0;
+    if (folder.isFile())
+      return folder.length();
+    File[] filelist = folder.listFiles();
+    if (filelist == null) {
+      return 0;
+    }
+    for (int i = 0; i < filelist.length; i++) {
+      if (filelist[i].isDirectory()) {
+        foldersize += getFileSize(filelist[i]);
+      } else {
+        fileCount++; //Counting the total files
+        foldersize += filelist[i].length();
+      }
+    }
+    return foldersize;    
+  }
+
+  public static void main(String[] args) {
+    if (Shell.WINDOWS)
+      System.out.println("Windows: "+ DUHelper.getFolderUsage(args[0]));
+    else
+      System.out.println("Other: " + DUHelper.getFolderUsage(args[0]));
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java

@@ -1326,7 +1326,7 @@ public final class FileContext {
    * 
    * 2. Partially qualified URIs (eg scheme but no host)
    * 
-   * fs:///A/B/file  Resolved according to the target file sytem. Eg resolving
+   * fs:///A/B/file  Resolved according to the target file system. Eg resolving
    *                 a symlink to hdfs:///A results in an exception because
    *                 HDFS URIs must be fully qualified, while a symlink to 
    *                 file:///A will not since Hadoop's local file systems 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -1864,7 +1864,7 @@ public abstract class FileSystem extends Configured implements Closeable {
    * 
    * Some file systems like LocalFileSystem have an initial workingDir
    * that we use as the starting workingDir. For other file systems
-   * like HDFS there is no built in notion of an inital workingDir.
+   * like HDFS there is no built in notion of an initial workingDir.
    * 
    * @return if there is built in notion of workingDir then it
    * is returned; else a null is returned.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java

@@ -19,18 +19,29 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import org.apache.commons.collections.map.CaseInsensitiveMap;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
@@ -46,6 +57,13 @@ public class FileUtil {
 
   private static final Log LOG = LogFactory.getLog(FileUtil.class);
 
+  /* The error code is defined in winutils to indicate insufficient
+   * privilege to create symbolic links. This value need to keep in
+   * sync with the constant of the same name in:
+   * "src\winutils\common.h"
+   * */
+  public static final int SYMLINK_NO_PRIVILEGE = 2;
+
   /**
    * convert an array of FileStatus to an array of Path
    * 
@@ -469,34 +487,6 @@ public class FileUtil {
     return dst;
   }
 
-  /**
-   * This class is only used on windows to invoke the cygpath command.
-   */
-  private static class CygPathCommand extends Shell {
-    String[] command;
-    String result;
-    CygPathCommand(String path) throws IOException {
-      command = new String[]{"cygpath", "-u", path};
-      run();
-    }
-    String getResult() throws IOException {
-      return result;
-    }
-    @Override
-    protected String[] getExecString() {
-      return command;
-    }
-    @Override
-    protected void parseExecResult(BufferedReader lines) throws IOException {
-      String line = lines.readLine();
-      if (line == null) {
-        throw new IOException("Can't convert '" + command[2] + 
-                              " to a cygwin path");
-      }
-      result = line;
-    }
-  }
-
   /**
    * Convert a os-native filename to a path that works for the shell.
    * @param filename The filename to convert
@@ -504,11 +494,7 @@ public class FileUtil {
    * @throws IOException on windows, there can be problems with the subprocess
    */
   public static String makeShellPath(String filename) throws IOException {
-    if (Path.WINDOWS) {
-      return new CygPathCommand(filename).getResult();
-    } else {
-      return filename;
-    }    
+    return filename;
   }
   
   /**
@@ -658,7 +644,7 @@ public class FileUtil {
     untarCommand.append(FileUtil.makeShellPath(untarDir)); 
     untarCommand.append("' ; ");
     untarCommand.append("tar -xf ");
-    
+
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
@@ -731,7 +717,7 @@ public class FileUtil {
   
   /**
    * Class for creating hardlinks.
-   * Supports Unix, Cygwin, WindXP.
+   * Supports Unix, WindXP.
    * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
    */
   @Deprecated
@@ -743,21 +729,67 @@ public class FileUtil {
 
   /**
    * Create a soft link between a src and destination
-   * only on a local disk. HDFS does not support this
+   * only on a local disk. HDFS does not support this.
+   * On Windows, when symlink creation fails due to security
+   * setting, we will log a warning. The return code in this
+   * case is 2.
    * @param target the target for symlink 
    * @param linkname the symlink
    * @return value returned by the command
    */
   public static int symLink(String target, String linkname) throws IOException{
-    String cmd = "ln -s " + target + " " + linkname;
-    Process p = Runtime.getRuntime().exec(cmd, null);
-    int returnVal = -1;
-    try{
-      returnVal = p.waitFor();
-    } catch(InterruptedException e){
-      //do nothing as of yet
+    // Run the input paths through Java's File so that they are converted to the
+    // native OS form
+    File targetFile = new File(target);
+    File linkFile = new File(linkname);
+
+    // If not on Java7+, copy a file instead of creating a symlink since
+    // Java6 has close to no support for symlinks on Windows. Specifically
+    // File#length and File#renameTo do not work as expected.
+    // (see HADOOP-9061 for additional details)
+    // We still create symlinks for directories, since the scenario in this
+    // case is different. The directory content could change in which
+    // case the symlink loses its purpose (for example task attempt log folder
+    // is symlinked under userlogs and userlogs are generated afterwards).
+    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
+      try {
+        LOG.info("FileUtil#symlink: On Java6, copying file instead "
+            + linkname + " -> " + target);
+        org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+      } catch (IOException ex) {
+        LOG.warn("FileUtil#symlink failed to copy the file with error: "
+            + ex.getMessage());
+        // Exit with non-zero exit code
+        return 1;
+      }
+      return 0;
     }
-    return returnVal;
+
+    String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
+        linkFile.getPath());
+    ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+    try {
+      shExec.execute();
+    } catch (Shell.ExitCodeException ec) {
+      int returnVal = ec.getExitCode();
+      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+        LOG.warn("Fail to create symbolic links on Windows. "
+            + "The default security settings in Windows disallow non-elevated "
+            + "administrators and all non-administrators from creating symbolic links. "
+            + "This behavior can be changed in the Local Security Policy management console");
+      } else if (returnVal != 0) {
+        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+            + returnVal + " with: " + ec.getMessage());
+      }
+      return returnVal;
+    } catch (IOException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error while create symlink " + linkname + " to " + target
+            + "." + " Exception: " + StringUtils.stringifyException(e));
+      }
+      throw e;
+    }
+    return shExec.getExitCode();
   }
   
   /**
@@ -781,30 +813,120 @@ public class FileUtil {
    * @param recursive true, if permissions should be changed recursively
    * @return the exit code from the command.
    * @throws IOException
-   * @throws InterruptedException
    */
   public static int chmod(String filename, String perm, boolean recursive)
-                            throws IOException, InterruptedException {
-    StringBuilder cmdBuf = new StringBuilder();
-    cmdBuf.append("chmod ");
-    if (recursive) {
-      cmdBuf.append("-R ");
-    }
-    cmdBuf.append(perm).append(" ");
-    cmdBuf.append(filename);
-    String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
-    ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+                            throws IOException {
+    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = new File(filename).getPath();
+    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
     try {
       shExec.execute();
-    }catch(Exception e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Error while changing permission : " + filename
-            + " Exception: ", e);
+    }catch(IOException e) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Error while changing permission : " + filename 
+                  +" Exception: " + StringUtils.stringifyException(e));
       }
     }
     return shExec.getExitCode();
   }
+
+  /**
+   * Set the ownership on a file / directory. User name and group name
+   * cannot both be null.
+   * @param file the file to change
+   * @param username the new user owner name
+   * @param groupname the new group owner name
+   * @throws IOException
+   */
+  public static void setOwner(File file, String username,
+      String groupname) throws IOException {
+    if (username == null && groupname == null) {
+      throw new IOException("username == null && groupname == null");
+    }
+    String arg = (username == null ? "" : username)
+        + (groupname == null ? "" : ":" + groupname);
+    String [] cmd = Shell.getSetOwnerCommand(arg);
+    execCommand(file, cmd);
+  }
+
+  /**
+   * Set permissions to the required value. Uses the java primitives instead
+   * of forking if group == other.
+   * @param f the file to change
+   * @param permission the new permissions
+   * @throws IOException
+   */
+  public static void setPermission(File f, FsPermission permission
+                                   ) throws IOException {
+    FsAction user = permission.getUserAction();
+    FsAction group = permission.getGroupAction();
+    FsAction other = permission.getOtherAction();
+
+    // use the native/fork if the group/other permissions are different
+    // or if the native is available or on Windows
+    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+      execSetPermission(f, permission);
+      return;
+    }
+    
+    boolean rv = true;
+    
+    // read perms
+    rv = f.setReadable(group.implies(FsAction.READ), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+      rv = f.setReadable(user.implies(FsAction.READ), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // write perms
+    rv = f.setWritable(group.implies(FsAction.WRITE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+      rv = f.setWritable(user.implies(FsAction.WRITE), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // exec perms
+    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+      rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
+      checkReturnValue(rv, f, permission);
+    }
+  }
+
+  private static void checkReturnValue(boolean rv, File p, 
+                                       FsPermission permission
+                                       ) throws IOException {
+    if (!rv) {
+      throw new IOException("Failed to set permissions of path: " + p + 
+                            " to " + 
+                            String.format("%04o", permission.toShort()));
+    }
+  }
+  
+  private static void execSetPermission(File f, 
+                                        FsPermission permission
+                                       )  throws IOException {
+    if (NativeIO.isAvailable()) {
+      NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
+    } else {
+      execCommand(f, Shell.getSetPermissionCommand(
+                  String.format("%04o", permission.toShort()), false));
+    }
+  }
   
+  static String execCommand(File f, String... cmd) throws IOException {
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = f.getCanonicalPath();
+    String output = Shell.execCommand(args);
+    return output;
+  }
+
   /**
    * Create a tmp file for a base file.
    * @param basefile the base file of the tmp
@@ -892,4 +1014,97 @@ public class FileUtil {
     }
     return fileNames;
   }  
+  
+  /**
+   * Create a jar file at the given path, containing a manifest with a classpath
+   * that references all specified entries.
+   * 
+   * Some platforms may have an upper limit on command line length.  For example,
+   * the maximum command line length on Windows is 8191 characters, but the
+   * length of the classpath may exceed this.  To work around this limitation,
+   * use this method to create a small intermediate jar with a manifest that
+   * contains the full classpath.  It returns the absolute path to the new jar,
+   * which the caller may set as the classpath for a new process.
+   * 
+   * Environment variable evaluation is not supported within a jar manifest, so
+   * this method expands environment variables before inserting classpath entries
+   * to the manifest.  The method parses environment variables according to
+   * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+   * environment variables are case-insensitive.  For example, %VAR% and %var%
+   * evaluate to the same value.
+   * 
+   * Specifying the classpath in a jar manifest does not support wildcards, so
+   * this method expands wildcards internally.  Any classpath entry that ends
+   * with * is translated to all files at that path with extension .jar or .JAR.
+   * 
+   * @param inputClassPath String input classpath to bundle into the jar manifest
+   * @param pwd Path to working directory to save jar
+   * @return String absolute path to new jar
+   * @throws IOException if there is an I/O error while writing the jar file
+   */
+  public static String createJarWithClassPath(String inputClassPath, Path pwd)
+      throws IOException {
+    // Replace environment variables, case-insensitive on Windows
+    @SuppressWarnings("unchecked")
+    Map<String, String> env = Shell.WINDOWS ?
+      new CaseInsensitiveMap(System.getenv()) : System.getenv();
+    String[] classPathEntries = inputClassPath.split(File.pathSeparator);
+    for (int i = 0; i < classPathEntries.length; ++i) {
+      classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
+        StringUtils.ENV_VAR_PATTERN, env);
+    }
+    File workingDir = new File(pwd.toString());
+    if (!workingDir.mkdirs()) {
+      // If mkdirs returns false because the working directory already exists,
+      // then this is acceptable.  If it returns false due to some other I/O
+      // error, then this method will fail later with an IOException while saving
+      // the jar.
+      LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
+    }
+
+    // Append all entries
+    List<String> classPathEntryList = new ArrayList<String>(
+      classPathEntries.length);
+    for (String classPathEntry: classPathEntries) {
+      if (classPathEntry.endsWith("*")) {
+        // Append all jars that match the wildcard
+        Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
+        FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
+          .globStatus(globPath);
+        if (wildcardJars != null) {
+          for (FileStatus wildcardJar: wildcardJars) {
+            classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
+              .toExternalForm());
+          }
+        }
+      } else {
+        // Append just this jar
+        classPathEntryList.add(new File(classPathEntry).toURI().toURL()
+          .toExternalForm());
+      }
+    }
+    String jarClassPath = StringUtils.join(" ", classPathEntryList);
+
+    // Create the manifest
+    Manifest jarManifest = new Manifest();
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+    jarManifest.getMainAttributes().putValue(
+        Attributes.Name.CLASS_PATH.toString(), jarClassPath);
+
+    // Write the manifest to output JAR file
+    File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
+    FileOutputStream fos = null;
+    BufferedOutputStream bos = null;
+    JarOutputStream jos = null;
+    try {
+      fos = new FileOutputStream(classPathJar);
+      bos = new BufferedOutputStream(fos);
+      jos = new JarOutputStream(bos, jarManifest);
+    } finally {
+      IOUtils.cleanup(LOG, jos, bos, fos);
+    }
+
+    return classPathJar.getCanonicalPath();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java

@@ -25,9 +25,11 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Arrays;
 
+import org.apache.hadoop.util.Shell;
+
 /**
  * Class for creating hardlinks.
- * Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X.
+ * Supports Unix/Linux, Windows via winutils , and Mac OS X.
  * 
  * The HardLink class was formerly a static inner class of FSUtil,
  * and the methods provided were blatantly non-thread-safe.
@@ -41,7 +43,7 @@ public class HardLink {
 
   public enum OSType {
     OS_TYPE_UNIX,
-    OS_TYPE_WINXP,
+    OS_TYPE_WIN,
     OS_TYPE_SOLARIS,
     OS_TYPE_MAC,
     OS_TYPE_FREEBSD
@@ -56,7 +58,7 @@ public class HardLink {
   //methods without instantiating the HardLink object
   static { 
     osType = getOSType();
-    if (osType == OSType.OS_TYPE_WINXP) {
+    if (osType == OSType.OS_TYPE_WIN) {
       // Windows
       getHardLinkCommand = new HardLinkCGWin();
     } else {
@@ -80,14 +82,8 @@ public class HardLink {
   
   static private OSType getOSType() {
     String osName = System.getProperty("os.name");
-    if (osName.contains("Windows") &&
-            (osName.contains("XP") 
-            || osName.contains("2003") 
-            || osName.contains("Vista")
-            || osName.contains("Windows_7")
-            || osName.contains("Windows 7") 
-            || osName.contains("Windows7"))) {
-      return OSType.OS_TYPE_WINXP;
+    if (Shell.WINDOWS) {
+      return OSType.OS_TYPE_WIN;
     }
     else if (osName.contains("SunOS") 
             || osName.contains("Solaris")) {
@@ -258,11 +254,6 @@ public class HardLink {
   
   /**
    * Implementation of HardLinkCommandGetter class for Windows
-   * 
-   * Note that the linkCount shell command for Windows is actually
-   * a Cygwin shell command, and depends on ${cygwin}/bin
-   * being in the Windows PATH environment variable, so
-   * stat.exe can be found.
    */
   static class HardLinkCGWin extends HardLinkCommandGetter {
     //The Windows command getter impl class and its member fields are
@@ -270,14 +261,16 @@ public class HardLink {
     //unit testing (sort of) on non-Win servers
 
     static String[] hardLinkCommand = {
-                        "fsutil","hardlink","create", null, null};
+                        Shell.WINUTILS,"hardlink","create", null, null};
     static String[] hardLinkMultPrefix = {
                         "cmd","/q","/c","for", "%f", "in", "("};
     static String   hardLinkMultDir = "\\%f";
     static String[] hardLinkMultSuffix = {
-                        ")", "do", "fsutil", "hardlink", "create", null, 
+                        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
                         "%f", "1>NUL"};
-    static String[] getLinkCountCommand = {"stat","-c%h", null};
+    static String[] getLinkCountCommand = {
+                        Shell.WINUTILS, "hardlink",
+                        "stat", null};
     //Windows guarantees only 8K - 1 bytes cmd length.
     //Subtract another 64b to allow for Java 'exec' overhead
     static final int maxAllowedCmdArgLength = 8*1024 - 65;
@@ -328,12 +321,6 @@ public class HardLink {
       String[] buf = new String[getLinkCountCommand.length];
       System.arraycopy(getLinkCountCommand, 0, buf, 0, 
                        getLinkCountCommand.length);
-      //The linkCount command is actually a Cygwin shell command,
-      //not a Windows shell command, so we should use "makeShellPath()"
-      //instead of "getCanonicalPath()".  However, that causes another
-      //shell exec to "cygpath.exe", and "stat.exe" actually can handle
-      //DOS-style paths (it just prints a couple hundred bytes of warning
-      //to stderr), so we use the more efficient "getCanonicalPath()".
       buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
       return buf;
     }
@@ -354,7 +341,7 @@ public class HardLink {
       //add the fixed overhead of the hardLinkMult command 
       //(prefix, suffix, and Dir suffix)
       sum += ("cmd.exe /q /c for %f in ( ) do "
-              + "fsutil hardlink create \\%f %f 1>NUL ").length();
+              + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length();
       return sum;
     }
     
@@ -581,14 +568,10 @@ public class HardLink {
   /* Create an IOException for failing to get link count. */
   private static IOException createIOException(File f, String message,
       String error, int exitvalue, Exception cause) {
-    
-    final String winErrMsg = "; Windows errors in getLinkCount are often due "
-         + "to Cygwin misconfiguration";
 
     final String s = "Failed to get link count on file " + f
         + ": message=" + message
         + "; error=" + error
-        + ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "")
         + "; exit value=" + exitvalue;
     return (cause == null) ? new IOException(s) : new IOException(s, cause);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.regex.Pattern;
 
 import org.apache.avro.reflect.Stringable;
 import org.apache.commons.lang.StringUtils;
@@ -43,9 +44,17 @@ public class Path implements Comparable {
   
   public static final String CUR_DIR = ".";
   
-  static final boolean WINDOWS
+  public static final boolean WINDOWS
     = System.getProperty("os.name").startsWith("Windows");
 
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern hasUriScheme =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern hasDriveLetterSpecifier =
+      Pattern.compile("^/?[a-zA-Z]:");
+
   private URI uri;                                // a hierarchical uri
 
   /** Resolve a child path against a parent path. */
@@ -81,7 +90,7 @@ public class Path implements Comparable {
                resolved.getPath(), resolved.getFragment());
   }
 
-  private void checkPathArg( String path ) {
+  private void checkPathArg( String path ) throws IllegalArgumentException {
     // disallow construction of a Path from an empty string
     if ( path == null ) {
       throw new IllegalArgumentException(
@@ -95,15 +104,16 @@ public class Path implements Comparable {
   
   /** Construct a path from a String.  Path strings are URIs, but with
    * unescaped elements and some additional normalization. */
-  public Path(String pathString) {
+  public Path(String pathString) throws IllegalArgumentException {
     checkPathArg( pathString );
     
     // We can't use 'new URI(String)' directly, since it assumes things are
     // escaped, which we don't require of Paths. 
     
     // add a slash in front of paths with Windows drive letters
-    if (hasWindowsDrive(pathString, false))
-      pathString = "/"+pathString;
+    if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') {
+      pathString = "/" + pathString;
+    }
 
     // parse uri components
     String scheme = null;
@@ -151,22 +161,54 @@ public class Path implements Comparable {
   private void initialize(String scheme, String authority, String path,
       String fragment) {
     try {
-      this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
+      this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment)
         .normalize();
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }
   }
 
-  private String normalizePath(String path) {
-    // remove double slashes & backslashes
+  /**
+   * Merge 2 paths such that the second path is appended relative to the first.
+   * The returned path has the scheme and authority of the first path.  On
+   * Windows, the drive specification in the second path is discarded.
+   * 
+   * @param path1 Path first path
+   * @param path2 Path second path, to be appended relative to path1
+   * @return Path merged path
+   */
+  public static Path mergePaths(Path path1, Path path2) {
+    String path2Str = path2.toUri().getPath();
+    if(hasWindowsDrive(path2Str)) {
+      path2Str = path2Str.substring(path2Str.indexOf(':')+1);
+    }
+    return new Path(path1 + path2Str);
+  }
+
+  /**
+   * Normalize a path string to use non-duplicated forward slashes as
+   * the path separator and remove any trailing path separators.
+   * @param scheme Supplies the URI scheme. Used to deduce whether we
+   *               should replace backslashes or not.
+   * @param path Supplies the scheme-specific part
+   * @return Normalized path string.
+   */
+  private static String normalizePath(String scheme, String path) {
+    // Remove double forward slashes.
     path = StringUtils.replace(path, "//", "/");
-    if (Path.WINDOWS) {
+
+    // Remove backslashes if this looks like a Windows path. Avoid
+    // the substitution if it looks like a non-local URI.
+    if (WINDOWS &&
+        (hasWindowsDrive(path) ||
+         (scheme == null) ||
+         (scheme.isEmpty()) ||
+         (scheme.equals("file")))) {
       path = StringUtils.replace(path, "\\", "/");
     }
     
     // trim trailing slash from non-root path (ignoring windows drive)
-    int minLength = hasWindowsDrive(path, true) ? 4 : 1;
+    int minLength = hasWindowsDrive(path) ? 4 : 1;
     if (path.length() > minLength && path.endsWith("/")) {
       path = path.substring(0, path.length()-1);
     }
@@ -174,17 +216,29 @@ public class Path implements Comparable {
     return path;
   }
 
-  private boolean hasWindowsDrive(String path, boolean slashed) {
-    if (!WINDOWS) return false;
-    int start = slashed ? 1 : 0;
-    return
-      path.length() >= start+2 &&
-      (slashed ? path.charAt(0) == '/' : true) &&
-      path.charAt(start+1) == ':' &&
-      ((path.charAt(start) >= 'A' && path.charAt(start) <= 'Z') ||
-       (path.charAt(start) >= 'a' && path.charAt(start) <= 'z'));
+  private static boolean hasWindowsDrive(String path) {
+    return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find());
   }
 
+  /**
+   * Determine whether a given path string represents an absolute path on
+   * Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+   *
+   * @param pathString Supplies the path string to evaluate.
+   * @param slashed true if the given path is prefixed with "/".
+   * @return true if the supplied path looks like an absolute path with a Windows
+   * drive-specifier.
+   */
+  public static boolean isWindowsAbsolutePath(final String pathString,
+                                              final boolean slashed) {
+    int start = (slashed ? 1 : 0);
+
+    return
+        hasWindowsDrive(pathString) &&
+        pathString.length() >= (start + 3) &&
+        ((pathString.charAt(start + 2) == SEPARATOR_CHAR) ||
+          (pathString.charAt(start + 2) == '\\'));
+  }
 
   /** Convert this to a URI. */
   public URI toUri() { return uri; }
@@ -207,7 +261,7 @@ public class Path implements Comparable {
    *  True if the path component (i.e. directory) of this URI is absolute.
    */
   public boolean isUriPathAbsolute() {
-    int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0;
+    int start = hasWindowsDrive(uri.getPath()) ? 3 : 0;
     return uri.getPath().startsWith(SEPARATOR, start);
    }
   
@@ -241,7 +295,7 @@ public class Path implements Comparable {
   public Path getParent() {
     String path = uri.getPath();
     int lastSlash = path.lastIndexOf('/');
-    int start = hasWindowsDrive(path, true) ? 3 : 0;
+    int start = hasWindowsDrive(path) ? 3 : 0;
     if ((path.length() == start) ||               // empty path
         (lastSlash == start && path.length() == start+1)) { // at root
       return null;
@@ -250,7 +304,7 @@ public class Path implements Comparable {
     if (lastSlash==-1) {
       parent = CUR_DIR;
     } else {
-      int end = hasWindowsDrive(path, true) ? 3 : 0;
+      int end = hasWindowsDrive(path) ? 3 : 0;
       parent = path.substring(0, lastSlash==end?end+1:lastSlash);
     }
     return new Path(uri.getScheme(), uri.getAuthority(), parent);
@@ -277,7 +331,7 @@ public class Path implements Comparable {
     if (uri.getPath() != null) {
       String path = uri.getPath();
       if (path.indexOf('/')==0 &&
-          hasWindowsDrive(path, true) &&          // has windows drive
+          hasWindowsDrive(path) &&                // has windows drive
           uri.getScheme() == null &&              // but no scheme
           uri.getAuthority() == null)             // or authority
         path = path.substring(1);                 // remove slash before drive
@@ -364,7 +418,7 @@ public class Path implements Comparable {
     URI newUri = null;
     try {
       newUri = new URI(scheme, authority , 
-        normalizePath(pathUri.getPath()), null, fragment);
+        normalizePath(scheme, pathUri.getPath()), null, fragment);
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java

@@ -43,7 +43,7 @@ public interface PositionedReadable {
     throws IOException;
   
   /**
-   * Read number of bytes equalt to the length of the buffer, from a given
+   * Read number of bytes equal to the length of the buffer, from a given
    * position within a file. This does not
    * change the current offset of a file, and is thread-safe.
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -508,9 +508,10 @@ public class RawLocalFileSystem extends FileSystem {
       return !super.getOwner().isEmpty(); 
     }
     
-    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
+    RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
-            f.lastModified(), fs.makeQualified(new Path(f.getPath())));
+          f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
+            fs.getWorkingDirectory()));
     }
     
     @Override
@@ -541,9 +542,10 @@ public class RawLocalFileSystem extends FileSystem {
     private void loadPermissionInfo() {
       IOException e = null;
       try {
-        StringTokenizer t = new StringTokenizer(
-            execCommand(new File(getPath().toUri()), 
-                        Shell.getGET_PERMISSION_COMMAND()));
+        String output = FileUtil.execCommand(new File(getPath().toUri()), 
+            Shell.getGetPermissionCommand());
+        StringTokenizer t =
+            new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -552,7 +554,17 @@ public class RawLocalFileSystem extends FileSystem {
         }
         setPermission(FsPermission.valueOf(permission));
         t.nextToken();
-        setOwner(t.nextToken());
+
+        String owner = t.nextToken();
+        // If on windows domain, token format is DOMAIN\\user and we want to
+        // extract only the user name
+        if (Shell.WINDOWS) {
+          int i = owner.indexOf('\\');
+          if (i != -1)
+            owner = owner.substring(i + 1);
+        }
+        setOwner(owner);
+
         setGroup(t.nextToken());
       } catch (Shell.ExitCodeException ioe) {
         if (ioe.getExitCode() != 1) {
@@ -588,17 +600,7 @@ public class RawLocalFileSystem extends FileSystem {
   @Override
   public void setOwner(Path p, String username, String groupname)
     throws IOException {
-    if (username == null && groupname == null) {
-      throw new IOException("username == null && groupname == null");
-    }
-
-    if (username == null) {
-      execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
-    } else {
-      //OWNER[:[GROUP]]
-      String s = username + (groupname == null? "": ":" + groupname);
-      execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
-    }
+    FileUtil.setOwner(pathToFile(p), username, groupname);
   }
 
   /**
@@ -608,20 +610,12 @@ public class RawLocalFileSystem extends FileSystem {
   public void setPermission(Path p, FsPermission permission)
     throws IOException {
     if (NativeIO.isAvailable()) {
-      NativeIO.chmod(pathToFile(p).getCanonicalPath(),
+      NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                      permission.toShort());
     } else {
-      execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
-          String.format("%05o", permission.toShort()));
+      String perm = String.format("%04o", permission.toShort());
+      Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+        FileUtil.makeShellPath(pathToFile(p), true)));
     }
   }
-
-  private static String execCommand(File f, String... cmd) throws IOException {
-    String[] args = new String[cmd.length + 1];
-    System.arraycopy(cmd, 0, args, 0, cmd.length);
-    args[cmd.length] = FileUtil.makeShellPath(f, true);
-    String output = Shell.execCommand(args);
-    return output;
-  }
-
-}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java

@@ -79,19 +79,17 @@ public abstract class TrashPolicy extends Configured {
 
   /**
    * Get an instance of the configured TrashPolicy based on the value 
-   * of the configuration paramater fs.trash.classname.
+   * of the configuration parameter fs.trash.classname.
    *
    * @param conf the configuration to be used
    * @param fs the file system to be used
    * @param home the home directory
    * @return an instance of TrashPolicy
    */
-  public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home)
-      throws IOException {
-    Class<? extends TrashPolicy> trashClass = conf.getClass("fs.trash.classname",
-                                                      TrashPolicyDefault.class,
-                                                      TrashPolicy.class);
-    TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf);
+  public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
+    Class<? extends TrashPolicy> trashClass = conf.getClass(
+        "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+    TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
     trash.initialize(conf, fs, home); // initialize TrashPolicy
     return trash;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java

@@ -92,7 +92,7 @@ public class TrashPolicyDefault extends TrashPolicy {
   }
   
   private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
-    return new Path(basePath + rmFilePath.toUri().getPath());
+    return Path.mergePaths(basePath, rmFilePath);
   }
 
   @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java

@@ -89,11 +89,9 @@ public class RawLocalFs extends DelegateToFileSystem {
     }
     // NB: Use createSymbolicLink in java.nio.file.Path once available
     try {
-      Shell.execCommand(Shell.LINK_COMMAND, "-s",
-                        new URI(target.toString()).getPath(),
-                        new URI(link.toString()).getPath());
-    } catch (URISyntaxException x) {
-      throw new IOException("Invalid symlink path: "+x.getMessage());
+      Shell.execCommand(Shell.getSymlinkCommand(
+        getPathWithoutSchemeAndAuthority(target),
+        getPathWithoutSchemeAndAuthority(link)));
     } catch (IOException x) {
       throw new IOException("Unable to create symlink: "+x.getMessage());
     }
@@ -176,4 +174,13 @@ public class RawLocalFs extends DelegateToFileSystem {
      */
     throw new AssertionError();
   }
+
+  private static String getPathWithoutSchemeAndAuthority(Path path) {
+    // This code depends on Path.toString() to remove the leading slash before
+    // the drive specification on Windows.
+    Path newPath = path.isUriPathAbsolute() ?
+      new Path(null, null, path.toUri().getPath()) :
+      path;
+    return newPath.toString();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.shell;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -72,8 +74,12 @@ abstract class CommandWithDestination extends FsCommand {
    */
   protected void getLocalDestination(LinkedList<String> args)
   throws IOException {
-    String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
-    dst = new PathData(new File(pathString), getConf());
+    try {
+      String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
+      dst = new PathData(new URI(pathString), getConf());
+    } catch (URISyntaxException e) {
+      throw new IOException("unexpected URISyntaxException", e);
+    }
   }
 
   /**
@@ -295,4 +301,4 @@ abstract class CommandWithDestination extends FsCommand {
       processDeleteOnExit();
     }
   }
-}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -60,16 +62,20 @@ class CopyCommands {
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
-      cf.parse(args);
+      try {
+        CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
+        cf.parse(args);
 
-      delimiter = cf.getOpt("nl") ? "\n" : null;
+        delimiter = cf.getOpt("nl") ? "\n" : null;
 
-      dst = new PathData(new File(args.removeLast()), getConf());
-      if (dst.exists && dst.stat.isDirectory()) {
-        throw new PathIsDirectoryException(dst.toString());
+        dst = new PathData(new URI(args.removeLast()), getConf());
+        if (dst.exists && dst.stat.isDirectory()) {
+          throw new PathIsDirectoryException(dst.toString());
+        }
+        srcs = new LinkedList<PathData>();
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
       }
-      srcs = new LinkedList<PathData>();
     }
 
     @Override
@@ -188,9 +194,13 @@ class CopyCommands {
     // commands operating on local paths have no need for glob expansion
     @Override
     protected List<PathData> expandArgument(String arg) throws IOException {
-      List<PathData> items = new LinkedList<PathData>();
-      items.add(new PathData(new File(arg), getConf()));
-      return items;
+      try {
+        List<PathData> items = new LinkedList<PathData>();
+        items.add(new PathData(new URI(arg), getConf()));
+        return items;
+      } catch (URISyntaxException e) {
+        throw new IOException("unexpected URISyntaxException", e);
+      }
     }
 
     @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -47,13 +48,14 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.JsonEncoding;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.util.MinimalPrettyPrinter;
 
 /**
- * Display contents of files 
+ * Display contents or checksums of files 
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -62,6 +64,7 @@ class Display extends FsCommand {
   public static void registerCommands(CommandFactory factory) {
     factory.addClass(Cat.class, "-cat");
     factory.addClass(Text.class, "-text");
+    factory.addClass(Checksum.class, "-checksum");
   }
 
   /**
@@ -161,6 +164,36 @@ class Display extends FsCommand {
       return i;
     }
   }
+  
+  public static class Checksum extends Display {
+    public static final String NAME = "checksum";
+    public static final String USAGE = "<src> ...";
+    public static final String DESCRIPTION =
+      "Dump checksum information for files that match the file\n" +
+      "pattern <src> to stdout. Note that this requires a round-trip\n" +
+      "to a datanode storing each block of the file, and thus is not\n" +
+      "efficient to run on a large number of files. The checksum of a\n" +
+      "file depends on its content, block size and the checksum\n" +
+      "algorithm and parameters used for creating the file.";
+
+    @Override
+    protected void processPath(PathData item) throws IOException {
+      if (item.stat.isDirectory()) {
+        throw new PathIsDirectoryException(item.toString());
+      }
+
+      FileChecksum checksum = item.fs.getFileChecksum(item.path);
+      if (checksum == null) {
+        out.printf("%s\tNONE\t\n", item.toString());
+      } else {
+        String checksumString = StringUtils.byteToHexString(
+            checksum.getBytes(), 0, checksum.getLength());
+        out.printf("%s\t%s\t%s\n",
+            item.toString(), checksum.getAlgorithmName(),
+            checksumString);
+      }
+    }
+  }
 
   protected class TextRecordInputStream extends InputStream {
     SequenceFile.Reader r;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java

@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Arrays;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathNotFoundException;
 
 /**
  * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
+ * PathData ensures that the returned path string will be the same as the
+ * one passed in during initialization (unlike Path objects which can
+ * modify the path string).
  * The stat field will be null if the path does not exist.
  */
 @InterfaceAudience.Private
@@ -51,6 +55,20 @@ public class PathData implements Comparable<PathData> {
   public FileStatus stat;
   public boolean exists;
 
+  /* True if the URI scheme was not present in the pathString but inferred.
+   */
+  private boolean inferredSchemeFromPath = false;
+
+  /**
+   *  Pre-compiled regular expressions to detect path formats.
+   */
+  private static final Pattern potentialUri =
+      Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+  private static final Pattern windowsNonUriAbsolutePath1 =
+      Pattern.compile("^/?[a-zA-Z]:\\\\");
+  private static final Pattern windowsNonUriAbsolutePath2 =
+      Pattern.compile("^/?[a-zA-Z]:/");
+
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
@@ -67,12 +85,12 @@ public class PathData implements Comparable<PathData> {
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
    * return exactly the same string used to initialize it
-   * @param localPath a local File
+   * @param localPath a local URI
    * @param conf the configuration file
    * @throws IOException if anything goes wrong...
    */
-  public PathData(File localPath, Configuration conf) throws IOException {
-    this(FileSystem.getLocal(conf), localPath.toString());
+  public PathData(URI localPath, Configuration conf) throws IOException {
+    this(FileSystem.getLocal(conf), localPath.getPath());
   }
 
   /**
@@ -86,6 +104,39 @@ public class PathData implements Comparable<PathData> {
     this(fs, pathString, lookupStat(fs, pathString, true));
   }
 
+  /**
+   * Validates the given Windows path.
+   * Throws IOException on failure.
+   * @param pathString a String of the path suppliued by the user.
+   */
+  private void ValidateWindowsPath(String pathString)
+  throws IOException
+  {
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      inferredSchemeFromPath = true;
+      return;
+    }
+
+    // Does it look like a URI? If so then just leave it alone.
+    if (potentialUri.matcher(pathString).find()) {
+      return;
+    }
+
+    // Looks like a relative path on Windows.
+    return;
+  }
+
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
@@ -100,6 +151,10 @@ public class PathData implements Comparable<PathData> {
     this.uri = stringToUri(pathString);
     this.path = fs.makeQualified(new Path(uri));
     setStat(stat);
+
+    if (Path.WINDOWS) {
+      ValidateWindowsPath(pathString);
+    }
   }
 
   // need a static method for the ctor above
@@ -236,7 +291,7 @@ public class PathData implements Comparable<PathData> {
    * Given a child of this directory, use the directory's path and the child's
    * basename to construct the string to the child.  This preserves relative
    * paths since Path will fully qualify.
-   * @param child a path contained within this directory
+   * @param childPath a path contained within this directory
    * @return String of the path relative to this directory
    */
   private String getStringForChildPath(Path childPath) {
@@ -386,7 +441,14 @@ public class PathData implements Comparable<PathData> {
     // No interpretation of symbols. Just decode % escaped chars.
     String decodedRemainder = uri.getSchemeSpecificPart();
 
-    if (scheme == null) {
+    // Drop the scheme if it was inferred to ensure fidelity between
+    // the input and output path strings.
+    if ((scheme == null) || (inferredSchemeFromPath)) {
+      if (Path.isWindowsAbsolutePath(decodedRemainder, true)) {
+        // Strip the leading '/' added in stringToUri so users see a valid
+        // Windows path.
+        decodedRemainder = decodedRemainder.substring(1);
+      }
       return decodedRemainder;
     } else {
       StringBuilder buffer = new StringBuilder();
@@ -409,13 +471,56 @@ public class PathData implements Comparable<PathData> {
     return ((LocalFileSystem)fs).pathToFile(path);
   }
 
+  /** Normalize the given Windows path string. This does the following:
+   *    1. Adds "file:" scheme for absolute paths.
+   *    2. Ensures the scheme-specific part starts with '/' per RFC2396.
+   *    3. Replaces backslash path separators with forward slashes.
+   *    @param pathString Path string supplied by the user.
+   *    @return normalized absolute path string. Returns the input string
+   *            if it is not a Windows absolute path.
+   */
+  private static String normalizeWindowsPath(String pathString)
+  throws IOException
+  {
+    if (!Path.WINDOWS) {
+      return pathString;
+    }
+
+    boolean slashed =
+        ((pathString.length() >= 1) && (pathString.charAt(0) == '/'));
+
+    // Is it a backslash-separated absolute path?
+    if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+      // Forward slashes disallowed in a backslash-separated path.
+      if (pathString.indexOf('/') != -1) {
+        throw new IOException("Invalid path string " + pathString);
+      }
+
+      pathString = pathString.replace('\\', '/');
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a forward slash-separated absolute path?
+    if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+      return "file:" + (slashed ? "" : "/") + pathString;
+    }
+
+    // Is it a backslash-separated relative file path (no scheme and
+    // no drive-letter specifier)?
+    if ((pathString.indexOf(':') == -1) && (pathString.indexOf('\\') != -1)) {
+      pathString = pathString.replace('\\', '/');
+    }
+
+    return pathString;
+  }
+
   /** Construct a URI from a String with unescaped special characters
-   *  that have non-standard sematics. e.g. /, ?, #. A custom parsing
-   *  is needed to prevent misbihaviors.
+   *  that have non-standard semantics. e.g. /, ?, #. A custom parsing
+   *  is needed to prevent misbehavior.
    *  @param pathString The input path in string form
    *  @return URI
    */
-  private static URI stringToUri(String pathString) {
+  private static URI stringToUri(String pathString) throws IOException {
     // We can't use 'new URI(String)' directly. Since it doesn't do quoting
     // internally, the internal parser may fail or break the string at wrong
     // places. Use of multi-argument ctors will quote those chars for us,
@@ -424,9 +529,10 @@ public class PathData implements Comparable<PathData> {
     // parse uri components
     String scheme = null;
     String authority = null;
-
     int start = 0;
 
+    pathString = normalizeWindowsPath(pathString);
+
     // parse uri scheme, if any
     int colon = pathString.indexOf(':');
     int slash = pathString.indexOf('/');
@@ -445,8 +551,7 @@ public class PathData implements Comparable<PathData> {
       authority = pathString.substring(start, authEnd);
       start = authEnd;
     }
-
-    // uri path is the rest of the string. ? or # are not interpreated,
+    // uri path is the rest of the string. ? or # are not interpreted,
     // but any occurrence of them will be quoted by the URI ctor.
     String path = pathString.substring(start, pathString.length());
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java

@@ -61,6 +61,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
 import org.mortbay.io.Buffer;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Handler;
@@ -305,6 +306,13 @@ public class HttpServer implements FilterContainer {
     ret.setAcceptQueueSize(128);
     ret.setResolveNames(false);
     ret.setUseDirectBuffers(false);
+    if(Shell.WINDOWS) {
+      // result of setting the SO_REUSEADDR flag is different on Windows
+      // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+      // without this 2 NN's can start on the same machine and listen on 
+      // the same port with indeterminate routing of incoming requests to them
+      ret.setReuseAddress(false);
+    }
     ret.setHeaderBufferSize(1024*64);
     return ret;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java

@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 
 /** 
  * A byte sequence that is usable as a key or value.
- * It is resizable and distinguishes between the size of the seqeunce and
+ * It is resizable and distinguishes between the size of the sequence and
  * the current capacity. The hash function is the front of the md5 of the 
  * buffer. The sort order is the same as memcmp.
  */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java

@@ -203,8 +203,8 @@ public class ReadaheadPool {
       // It's also possible that we'll end up requesting readahead on some
       // other FD, which may be wasted work, but won't cause a problem.
       try {
-        NativeIO.posixFadviseIfPossible(fd, off, len,
-            NativeIO.POSIX_FADV_WILLNEED);
+        NativeIO.POSIX.posixFadviseIfPossible(fd, off, len,
+            NativeIO.POSIX.POSIX_FADV_WILLNEED);
       } catch (IOException ioe) {
         if (canceled) {
           // no big deal - the reader canceled the request and closed

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java

@@ -22,6 +22,7 @@ import java.io.FileDescriptor;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,7 +31,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.nativeio.Errno;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIOException;
-import org.apache.hadoop.io.nativeio.NativeIO.Stat;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
@@ -120,7 +121,7 @@ public class SecureIOUtils {
     FileInputStream fis = new FileInputStream(f);
     boolean success = false;
     try {
-      Stat stat = NativeIO.getFstat(fis.getFD());
+      Stat stat = NativeIO.POSIX.getFstat(fis.getFD());
       checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
           expectedGroup);
       success = true;
@@ -166,35 +167,30 @@ public class SecureIOUtils {
     if (skipSecurity) {
       return insecureCreateForWrite(f, permissions);
     } else {
-      // Use the native wrapper around open(2)
-      try {
-        FileDescriptor fd = NativeIO.open(f.getAbsolutePath(),
-          NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL,
-          permissions);
-        return new FileOutputStream(fd);
-      } catch (NativeIOException nioe) {
-        if (nioe.getErrno() == Errno.EEXIST) {
-          throw new AlreadyExistsException(nioe);
-        }
-        throw nioe;
-      }
+      return NativeIO.getCreateForWriteFileOutputStream(f, permissions);
     }
   }
 
   private static void checkStat(File f, String owner, String group, 
       String expectedOwner, 
       String expectedGroup) throws IOException {
+    boolean success = true;
     if (expectedOwner != null &&
         !expectedOwner.equals(owner)) {
-      throw new IOException(
-        "Owner '" + owner + "' for path " + f + " did not match " +
-        "expected owner '" + expectedOwner + "'");
+      if (Path.WINDOWS) {
+        UserGroupInformation ugi =
+            UserGroupInformation.createRemoteUser(expectedOwner);
+        final String adminsGroupString = "Administrators";
+        success = owner.equals(adminsGroupString)
+            && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString);
+      } else {
+        success = false;
+      }
     }
-    if (expectedGroup != null &&
-        !expectedGroup.equals(group)) {
+    if (!success) {
       throw new IOException(
-        "Group '" + group + "' for path " + f + " did not match " +
-        "expected group '" + expectedGroup + "'");
+          "Owner '" + owner + "' for path " + f + " did not match " +
+              "expected owner '" + expectedOwner + "'");
     }
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java

@@ -128,7 +128,7 @@ public class Text extends BinaryComparable
   /**
    * Returns the Unicode Scalar Value (32-bit integer value)
    * for the character at <code>position</code>. Note that this
-   * method avoids using the converter or doing String instatiation
+   * method avoids using the converter or doing String instantiation
    * @return the Unicode scalar value at position or -1
    *          if the position is invalid or points to a
    *          trailing byte
@@ -527,7 +527,7 @@ public class Text extends BinaryComparable
     int length = 0;
     int state = LEAD_BYTE;
     while (count < start+len) {
-      int aByte = ((int) utf8[count] & 0xFF);
+      int aByte = utf8[count] & 0xFF;
 
       switch (state) {
       case LEAD_BYTE:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java

@@ -23,108 +23,156 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Seekable;
 import org.apache.hadoop.io.compress.bzip2.BZip2Constants;
-import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor;
-import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor;
 import org.apache.hadoop.io.compress.bzip2.CBZip2InputStream;
 import org.apache.hadoop.io.compress.bzip2.CBZip2OutputStream;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
 
 /**
- * This class provides CompressionOutputStream and CompressionInputStream for
- * compression and decompression. Currently we dont have an implementation of
- * the Compressor and Decompressor interfaces, so those methods of
- * CompressionCodec which have a Compressor or Decompressor type argument, throw
- * UnsupportedOperationException.
+ * This class provides output and input streams for bzip2 compression
+ * and decompression.  It uses the native bzip2 library on the system
+ * if possible, else it uses a pure-Java implementation of the bzip2
+ * algorithm.  The configuration parameter
+ * io.compression.codec.bzip2.library can be used to control this
+ * behavior.
+ *
+ * In the pure-Java mode, the Compressor and Decompressor interfaces
+ * are not implemented.  Therefore, in that mode, those methods of
+ * CompressionCodec which have a Compressor or Decompressor type
+ * argument, throw UnsupportedOperationException.
+ *
+ * Currently, support for splittability is available only in the
+ * pure-Java mode; therefore, if a SplitCompressionInputStream is
+ * requested, the pure-Java implementation is used, regardless of the
+ * setting of the configuration parameter mentioned above.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public class BZip2Codec implements SplittableCompressionCodec {
+public class BZip2Codec implements Configurable, SplittableCompressionCodec {
 
   private static final String HEADER = "BZ";
   private static final int HEADER_LEN = HEADER.length();
   private static final String SUB_HEADER = "h9";
   private static final int SUB_HEADER_LEN = SUB_HEADER.length();
 
+  private Configuration conf;
+  
   /**
-  * Creates a new instance of BZip2Codec
+   * Set the configuration to be used by this object.
+   *
+   * @param conf the configuration object.
+   */
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+  
+  /**
+   * Return the configuration used by this object.
+   *
+   * @return the configuration object used by this objec.
+   */
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+  
+  /**
+  * Creates a new instance of BZip2Codec.
   */
   public BZip2Codec() { }
 
   /**
-  * Creates CompressionOutputStream for BZip2
-  *
-  * @param out
-  *            The output Stream
-  * @return The BZip2 CompressionOutputStream
-  * @throws java.io.IOException
-  *             Throws IO exception
-  */
+   * Create a {@link CompressionOutputStream} that will write to the given
+   * {@link OutputStream}.
+   *
+   * @param out        the location for the final output stream
+   * @return a stream the user can write uncompressed data to, to have it 
+   *         compressed
+   * @throws IOException
+   */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
       throws IOException {
-    return new BZip2CompressionOutputStream(out);
+    return createOutputStream(out, createCompressor());
   }
 
   /**
-  * Creates a compressor using given OutputStream.
+   * Create a {@link CompressionOutputStream} that will write to the given
+   * {@link OutputStream} with the given {@link Compressor}.
    *
-  * @return CompressionOutputStream
-    @throws java.io.IOException
+   * @param out        the location for the final output stream
+   * @param compressor compressor to use
+   * @return a stream the user can write uncompressed data to, to have it 
+   *         compressed
+   * @throws IOException
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
       Compressor compressor) throws IOException {
-    return createOutputStream(out);
+    return Bzip2Factory.isNativeBzip2Loaded(conf) ?
+      new CompressorStream(out, compressor, 
+                           conf.getInt("io.file.buffer.size", 4*1024)) :
+      new BZip2CompressionOutputStream(out);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return BZip2DummyCompressor.class
-  */
+   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+   *
+   * @return the type of compressor needed by this codec.
+   */
   @Override
-  public Class<? extends org.apache.hadoop.io.compress.Compressor> getCompressorType() {
-    return BZip2DummyCompressor.class;
+  public Class<? extends Compressor> getCompressorType() {
+    return Bzip2Factory.getBzip2CompressorType(conf);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return Compressor
-  */
+   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+   *
+   * @return a new compressor for use by this codec
+   */
   @Override
   public Compressor createCompressor() {
-    return new BZip2DummyCompressor();
+    return Bzip2Factory.getBzip2Compressor(conf);
   }
 
   /**
-  * Creates CompressionInputStream to be used to read off uncompressed data.
-  *
-  * @param in
-  *            The InputStream
-  * @return Returns CompressionInputStream for BZip2
-  * @throws java.io.IOException
-  *             Throws IOException
-  */
+   * Create a {@link CompressionInputStream} that will read from the given
+   * input stream and return a stream for uncompressed data.
+   *
+   * @param in the stream to read compressed bytes from
+   * @return a stream to read uncompressed bytes from
+   * @throws IOException
+   */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
       throws IOException {
-    return new BZip2CompressionInputStream(in);
+    return createInputStream(in, createDecompressor());
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return CompressionInputStream
-  */
+   * Create a {@link CompressionInputStream} that will read from the given
+   * {@link InputStream} with the given {@link Decompressor}, and return a 
+   * stream for uncompressed data.
+   *
+   * @param in           the stream to read compressed bytes from
+   * @param decompressor decompressor to use
+   * @return a stream to read uncompressed bytes from
+   * @throws IOException
+   */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
       Decompressor decompressor) throws IOException {
-    return createInputStream(in);
+    return Bzip2Factory.isNativeBzip2Loaded(conf) ? 
+      new DecompressorStream(in, decompressor,
+                             conf.getInt("io.file.buffer.size", 4*1024)) :
+      new BZip2CompressionInputStream(in);
   }
 
   /**
@@ -139,7 +187,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
    *
    * @return CompressionInputStream for BZip2 aligned at block boundaries
    */
-  @Override
   public SplitCompressionInputStream createInputStream(InputStream seekableIn,
       Decompressor decompressor, long start, long end, READ_MODE readMode)
       throws IOException {
@@ -184,23 +231,23 @@ public class BZip2Codec implements SplittableCompressionCodec {
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return BZip2DummyDecompressor.class
-  */
+   * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+   *
+   * @return the type of decompressor needed by this codec.
+   */
   @Override
-  public Class<? extends org.apache.hadoop.io.compress.Decompressor> getDecompressorType() {
-    return BZip2DummyDecompressor.class;
+  public Class<? extends Decompressor> getDecompressorType() {
+    return Bzip2Factory.getBzip2DecompressorType(conf);
   }
 
   /**
-  * This functionality is currently not supported.
-  *
-  * @return Decompressor
-  */
+   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+   *
+   * @return a new decompressor for use by this codec
+   */
   @Override
   public Decompressor createDecompressor() {
-    return new BZip2DummyDecompressor();
+    return Bzip2Factory.getBzip2Decompressor(conf);
   }
 
   /**
@@ -236,7 +283,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }
 
-    @Override
     public void finish() throws IOException {
       if (needsReset) {
         // In the case that nothing is written to this stream, we still need to
@@ -256,14 +302,12 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }    
     
-    @Override
     public void resetState() throws IOException {
       // Cannot write to out at this point because out might not be ready
       // yet, as in SequenceFile.Writer implementation.
       needsReset = true;
     }
 
-    @Override
     public void write(int b) throws IOException {
       if (needsReset) {
         internalReset();
@@ -271,7 +315,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       this.output.write(b);
     }
 
-    @Override
     public void write(byte[] b, int off, int len) throws IOException {
       if (needsReset) {
         internalReset();
@@ -279,7 +322,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       this.output.write(b, off, len);
     }
 
-    @Override
     public void close() throws IOException {
       if (needsReset) {
         // In the case that nothing is written to this stream, we still need to
@@ -397,7 +439,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
 
     }// end of method
 
-    @Override
     public void close() throws IOException {
       if (!needsReset) {
         input.close();
@@ -433,7 +474,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
     *
     */
 
-    @Override
     public int read(byte[] b, int off, int len) throws IOException {
       if (needsReset) {
         internalReset();
@@ -457,7 +497,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
 
     }
 
-    @Override
     public int read() throws IOException {
       byte b[] = new byte[1];
       int result = this.read(b, 0, 1);
@@ -472,7 +511,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       }
     }    
     
-    @Override
     public void resetState() throws IOException {
       // Cannot read from bufferedIn at this point because bufferedIn
       // might not be ready
@@ -480,7 +518,6 @@ public class BZip2Codec implements SplittableCompressionCodec {
       needsReset = true;
     }
 
-    @Override
     public long getPos() {
       return this.compressedStreamPosition;
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java

@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.Compressor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A {@link Compressor} based on the popular 
+ * bzip2 compression algorithm.
+ * http://www.bzip2.org/
+ * 
+ */
+public class Bzip2Compressor implements Compressor {
+  private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024;
+
+  // The default values for the block size and work factor are the same 
+  // those in Julian Seward's original bzip2 implementation.
+  static final int DEFAULT_BLOCK_SIZE = 9;
+  static final int DEFAULT_WORK_FACTOR = 30;
+
+  private static final Log LOG = LogFactory.getLog(Bzip2Compressor.class);
+
+  // HACK - Use this as a global lock in the JNI layer.
+  private static Class<Bzip2Compressor> clazz = Bzip2Compressor.class;
+
+  private long stream;
+  private int blockSize;
+  private int workFactor;
+  private int directBufferSize;
+  private byte[] userBuf = null;
+  private int userBufOff = 0, userBufLen = 0;
+  private Buffer uncompressedDirectBuf = null;
+  private int uncompressedDirectBufOff = 0, uncompressedDirectBufLen = 0;
+  private boolean keepUncompressedBuf = false;
+  private Buffer compressedDirectBuf = null;
+  private boolean finish, finished;
+
+  /**
+   * Creates a new compressor with a default values for the
+   * compression block size and work factor.  Compressed data will be
+   * generated in bzip2 format.
+   */
+  public Bzip2Compressor() {
+    this(DEFAULT_BLOCK_SIZE, DEFAULT_WORK_FACTOR, DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  /**
+   * Creates a new compressor, taking settings from the configuration.
+   */
+  public Bzip2Compressor(Configuration conf) {
+    this(Bzip2Factory.getBlockSize(conf),
+         Bzip2Factory.getWorkFactor(conf),
+         DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  /** 
+   * Creates a new compressor using the specified block size.
+   * Compressed data will be generated in bzip2 format.
+   * 
+   * @param blockSize The block size to be used for compression.  This is
+   *        an integer from 1 through 9, which is multiplied by 100,000 to 
+   *        obtain the actual block size in bytes.
+   * @param workFactor This parameter is a threshold that determines when a 
+   *        fallback algorithm is used for pathological data.  It ranges from
+   *        0 to 250.
+   * @param directBufferSize Size of the direct buffer to be used.
+   */
+  public Bzip2Compressor(int blockSize, int workFactor, 
+                         int directBufferSize) {
+    this.blockSize = blockSize;
+    this.workFactor = workFactor;
+    this.directBufferSize = directBufferSize;
+    stream = init(blockSize, workFactor);
+    uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+  }
+
+  /**
+   * Prepare the compressor to be used in a new stream with settings defined in
+   * the given Configuration. It will reset the compressor's block size and
+   * and work factor.
+   * 
+   * @param conf Configuration storing new settings
+   */
+  @Override
+  public synchronized void reinit(Configuration conf) {
+    reset();
+    end(stream);
+    if (conf == null) {
+      stream = init(blockSize, workFactor);
+      return;
+    }
+    blockSize = Bzip2Factory.getBlockSize(conf);
+    workFactor = Bzip2Factory.getWorkFactor(conf);
+    stream = init(blockSize, workFactor);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Reinit compressor with new compression configuration");
+    }
+  }
+
+  @Override
+  public synchronized void setInput(byte[] b, int off, int len) {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    this.userBuf = b;
+    this.userBufOff = off;
+    this.userBufLen = len;
+    uncompressedDirectBufOff = 0;
+    setInputFromSavedData();
+    
+    // Reinitialize bzip2's output direct buffer.
+    compressedDirectBuf.limit(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+  }
+  
+  // Copy enough data from userBuf to uncompressedDirectBuf.
+  synchronized void setInputFromSavedData() {
+    int len = Math.min(userBufLen, uncompressedDirectBuf.remaining());
+    ((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff, len);
+    userBufLen -= len;
+    userBufOff += len;
+    uncompressedDirectBufLen = uncompressedDirectBuf.position();
+  }
+
+  @Override
+  public synchronized void setDictionary(byte[] b, int off, int len) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public synchronized boolean needsInput() {
+    // Compressed data still available?
+    if (compressedDirectBuf.remaining() > 0) {
+      return false;
+    }
+
+    // Uncompressed data available in either the direct buffer or user buffer?
+    if (keepUncompressedBuf && uncompressedDirectBufLen > 0)
+      return false;
+    
+    if (uncompressedDirectBuf.remaining() > 0) {
+      // Check if we have consumed all data in the user buffer.
+      if (userBufLen <= 0) {
+        return true;
+      } else {
+        // Copy enough data from userBuf to uncompressedDirectBuf.
+        setInputFromSavedData();
+        return uncompressedDirectBuf.remaining() > 0;
+      }
+    }
+    
+    return false;
+  }
+  
+  @Override
+  public synchronized void finish() {
+    finish = true;
+  }
+  
+  @Override
+  public synchronized boolean finished() {
+    // Check if bzip2 says it has finished and
+    // all compressed data has been consumed.
+    return (finished && compressedDirectBuf.remaining() == 0);
+  }
+
+  @Override
+  public synchronized int compress(byte[] b, int off, int len) 
+    throws IOException {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    // Check if there is compressed data.
+    int n = compressedDirectBuf.remaining();
+    if (n > 0) {
+      n = Math.min(n, len);
+      ((ByteBuffer)compressedDirectBuf).get(b, off, n);
+      return n;
+    }
+
+    // Re-initialize bzip2's output direct buffer.
+    compressedDirectBuf.rewind();
+    compressedDirectBuf.limit(directBufferSize);
+
+    // Compress the data.
+    n = deflateBytesDirect();
+    compressedDirectBuf.limit(n);
+    
+    // Check if bzip2 has consumed the entire input buffer.
+    // Set keepUncompressedBuf properly.
+    if (uncompressedDirectBufLen <= 0) { // bzip2 consumed all input
+      keepUncompressedBuf = false;
+      uncompressedDirectBuf.clear();
+      uncompressedDirectBufOff = 0;
+      uncompressedDirectBufLen = 0;
+    } else {
+      keepUncompressedBuf = true;
+    }
+
+    // Get at most 'len' bytes.
+    n = Math.min(n, len);
+    ((ByteBuffer)compressedDirectBuf).get(b, off, n);
+
+    return n;
+  }
+
+  /**
+   * Returns the total number of compressed bytes output so far.
+   *
+   * @return the total (non-negative) number of compressed bytes output so far
+   */
+  @Override
+  public synchronized long getBytesWritten() {
+    checkStream();
+    return getBytesWritten(stream);
+  }
+
+  /**
+   * Returns the total number of uncompressed bytes input so far.</p>
+   *
+   * @return the total (non-negative) number of uncompressed bytes input so far
+   */
+  @Override
+  public synchronized long getBytesRead() {
+    checkStream();
+    return getBytesRead(stream);
+  }
+
+  @Override
+  public synchronized void reset() {
+    checkStream();
+    end(stream);
+    stream = init(blockSize, workFactor);
+    finish = false;
+    finished = false;
+    uncompressedDirectBuf.rewind();
+    uncompressedDirectBufOff = uncompressedDirectBufLen = 0;
+    keepUncompressedBuf = false;
+    compressedDirectBuf.limit(directBufferSize);
+    compressedDirectBuf.position(directBufferSize);
+    userBufOff = userBufLen = 0;
+  }
+  
+  @Override
+  public synchronized void end() {
+    if (stream != 0) {
+      end(stream);
+      stream = 0;
+    }
+  }
+  
+  static void initSymbols(String libname) {
+    initIDs(libname);
+  }
+
+  private void checkStream() {
+    if (stream == 0)
+      throw new NullPointerException();
+  }
+  
+  private native static void initIDs(String libname);
+  private native static long init(int blockSize, int workFactor);
+  private native int deflateBytesDirect();
+  private native static long getBytesRead(long strm);
+  private native static long getBytesWritten(long strm);
+  private native static void end(long strm);
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java

@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.io.compress.Decompressor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A {@link Decompressor} based on the popular 
+ * bzip2 compression algorithm.
+ * http://www.bzip2.org/
+ * 
+ */
+public class Bzip2Decompressor implements Decompressor {
+  private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024;
+  
+  private static final Log LOG = LogFactory.getLog(Bzip2Decompressor.class);
+
+  // HACK - Use this as a global lock in the JNI layer.
+  private static Class<Bzip2Decompressor> clazz = Bzip2Decompressor.class;
+  
+  private long stream;
+  private boolean conserveMemory;
+  private int directBufferSize;
+  private Buffer compressedDirectBuf = null;
+  private int compressedDirectBufOff, compressedDirectBufLen;
+  private Buffer uncompressedDirectBuf = null;
+  private byte[] userBuf = null;
+  private int userBufOff = 0, userBufLen = 0;
+  private boolean finished;
+
+  /**
+   * Creates a new decompressor.
+   */
+  public Bzip2Decompressor(boolean conserveMemory, int directBufferSize) {
+    this.conserveMemory = conserveMemory;
+    this.directBufferSize = directBufferSize;
+    compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+    
+    stream = init(conserveMemory ? 1 : 0);
+  }
+  
+  public Bzip2Decompressor() {
+    this(false, DEFAULT_DIRECT_BUFFER_SIZE);
+  }
+
+  @Override
+  public synchronized void setInput(byte[] b, int off, int len) {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+  
+    this.userBuf = b;
+    this.userBufOff = off;
+    this.userBufLen = len;
+    
+    setInputFromSavedData();
+    
+    // Reinitialize bzip2's output direct buffer.
+    uncompressedDirectBuf.limit(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+  }
+  
+  synchronized void setInputFromSavedData() {
+    compressedDirectBufOff = 0;
+    compressedDirectBufLen = userBufLen;
+    if (compressedDirectBufLen > directBufferSize) {
+      compressedDirectBufLen = directBufferSize;
+    }
+
+    // Reinitialize bzip2's input direct buffer.
+    compressedDirectBuf.rewind();
+    ((ByteBuffer)compressedDirectBuf).put(userBuf, userBufOff, 
+                                          compressedDirectBufLen);
+    
+    // Note how much data is being fed to bzip2.
+    userBufOff += compressedDirectBufLen;
+    userBufLen -= compressedDirectBufLen;
+  }
+
+  @Override
+  public synchronized void setDictionary(byte[] b, int off, int len) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public synchronized boolean needsInput() {
+    // Consume remaining compressed data?
+    if (uncompressedDirectBuf.remaining() > 0) {
+      return false;
+    }
+    
+    // Check if bzip2 has consumed all input.
+    if (compressedDirectBufLen <= 0) {
+      // Check if we have consumed all user-input.
+      if (userBufLen <= 0) {
+        return true;
+      } else {
+        setInputFromSavedData();
+      }
+    }
+    
+    return false;
+  }
+
+  @Override
+  public synchronized boolean needsDictionary() {
+    return false;
+  }
+
+  @Override
+  public synchronized boolean finished() {
+    // Check if bzip2 says it has finished and
+    // all compressed data has been consumed.
+    return (finished && uncompressedDirectBuf.remaining() == 0);
+  }
+
+  @Override
+  public synchronized int decompress(byte[] b, int off, int len) 
+    throws IOException {
+    if (b == null) {
+      throw new NullPointerException();
+    }
+    if (off < 0 || len < 0 || off > b.length - len) {
+      throw new ArrayIndexOutOfBoundsException();
+    }
+    
+    // Check if there is uncompressed data.
+    int n = uncompressedDirectBuf.remaining();
+    if (n > 0) {
+      n = Math.min(n, len);
+      ((ByteBuffer)uncompressedDirectBuf).get(b, off, n);
+      return n;
+    }
+    
+    // Re-initialize bzip2's output direct buffer.
+    uncompressedDirectBuf.rewind();
+    uncompressedDirectBuf.limit(directBufferSize);
+
+    // Decompress the data.
+    n = finished ? 0 : inflateBytesDirect();
+    uncompressedDirectBuf.limit(n);
+
+    // Get at most 'len' bytes.
+    n = Math.min(n, len);
+    ((ByteBuffer)uncompressedDirectBuf).get(b, off, n);
+
+    return n;
+  }
+  
+  /**
+   * Returns the total number of uncompressed bytes output so far.
+   *
+   * @return the total (non-negative) number of uncompressed bytes output so far
+   */
+  public synchronized long getBytesWritten() {
+    checkStream();
+    return getBytesWritten(stream);
+  }
+
+  /**
+   * Returns the total number of compressed bytes input so far.</p>
+   *
+   * @return the total (non-negative) number of compressed bytes input so far
+   */
+  public synchronized long getBytesRead() {
+    checkStream();
+    return getBytesRead(stream);
+  }
+
+  /**
+   * Returns the number of bytes remaining in the input buffers; normally
+   * called when finished() is true to determine amount of post-gzip-stream
+   * data.</p>
+   *
+   * @return the total (non-negative) number of unprocessed bytes in input
+   */
+  @Override
+  public synchronized int getRemaining() {
+    checkStream();
+    return userBufLen + getRemaining(stream);  // userBuf + compressedDirectBuf
+  }
+
+  /**
+   * Resets everything including the input buffers (user and direct).</p>
+   */
+  @Override
+  public synchronized void reset() {
+    checkStream();
+    end(stream);
+    stream = init(conserveMemory ? 1 : 0);
+    finished = false;
+    compressedDirectBufOff = compressedDirectBufLen = 0;
+    uncompressedDirectBuf.limit(directBufferSize);
+    uncompressedDirectBuf.position(directBufferSize);
+    userBufOff = userBufLen = 0;
+  }
+
+  @Override
+  public synchronized void end() {
+    if (stream != 0) {
+      end(stream);
+      stream = 0;
+    }
+  }
+
+  static void initSymbols(String libname) {
+    initIDs(libname);
+  }
+
+  private void checkStream() {
+    if (stream == 0)
+      throw new NullPointerException();
+  }
+  
+  private native static void initIDs(String libname);
+  private native static long init(int conserveMemory);
+  private native int inflateBytesDirect();
+  private native static long getBytesRead(long strm);
+  private native static long getBytesWritten(long strm);
+  private native static int getRemaining(long strm);
+  private native static void end(long strm);
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Compressor;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor;
+import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor;
+import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor;
+
+/**
+ * A collection of factories to create the right 
+ * bzip2 compressor/decompressor instances.
+ * 
+ */
+public class Bzip2Factory {
+  private static final Log LOG = LogFactory.getLog(Bzip2Factory.class);
+
+  private static String bzip2LibraryName = "";
+  private static boolean nativeBzip2Loaded;
+  
+  /**
+   * Check if native-bzip2 code is loaded & initialized correctly and 
+   * can be loaded for this job.
+   * 
+   * @param conf configuration
+   * @return <code>true</code> if native-bzip2 is loaded & initialized 
+   *         and can be loaded for this job, else <code>false</code>
+   */
+  public static boolean isNativeBzip2Loaded(Configuration conf) {
+    String libname = conf.get("io.compression.codec.bzip2.library", 
+                              "system-native");
+    if (!bzip2LibraryName.equals(libname)) {
+      nativeBzip2Loaded = false;
+      bzip2LibraryName = libname;
+      if (libname.equals("java-builtin")) {
+        LOG.info("Using pure-Java version of bzip2 library");
+      } else if (conf.getBoolean(
+                CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, 
+                CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
+          NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          // Initialize the native library.
+          Bzip2Compressor.initSymbols(libname);
+          Bzip2Decompressor.initSymbols(libname);
+          nativeBzip2Loaded = true;
+          LOG.info("Successfully loaded & initialized native-bzip2 library " +
+                   libname);
+        } catch (Throwable t) {
+          LOG.warn("Failed to load/initialize native-bzip2 library " + 
+                   libname + ", will use pure-Java version");
+        }
+      }
+    }
+    return nativeBzip2Loaded;
+  }
+
+  /**
+   * Return the appropriate type of the bzip2 compressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate type of the bzip2 compressor.
+   */
+  public static Class<? extends Compressor> 
+  getBzip2CompressorType(Configuration conf) {
+    return isNativeBzip2Loaded(conf) ? 
+      Bzip2Compressor.class : BZip2DummyCompressor.class;
+  }
+  
+  /**
+   * Return the appropriate implementation of the bzip2 compressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate implementation of the bzip2 compressor.
+   */
+  public static Compressor getBzip2Compressor(Configuration conf) {
+    return isNativeBzip2Loaded(conf)? 
+      new Bzip2Compressor(conf) : new BZip2DummyCompressor();
+  }
+
+  /**
+   * Return the appropriate type of the bzip2 decompressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate type of the bzip2 decompressor.
+   */
+  public static Class<? extends Decompressor> 
+  getBzip2DecompressorType(Configuration conf) {
+    return  isNativeBzip2Loaded(conf) ? 
+      Bzip2Decompressor.class : BZip2DummyDecompressor.class;
+  }
+  
+  /**
+   * Return the appropriate implementation of the bzip2 decompressor. 
+   * 
+   * @param conf configuration
+   * @return the appropriate implementation of the bzip2 decompressor.
+   */
+  public static Decompressor getBzip2Decompressor(Configuration conf) {
+    return isNativeBzip2Loaded(conf) ? 
+      new Bzip2Decompressor() : new BZip2DummyDecompressor();
+  }
+
+  public static void setBlockSize(Configuration conf, int blockSize) {
+    conf.setInt("bzip2.compress.blocksize", blockSize);
+  }
+
+  public static int getBlockSize(Configuration conf) {
+    return conf.getInt("bzip2.compress.blocksize", 
+                       Bzip2Compressor.DEFAULT_BLOCK_SIZE);
+  }
+
+  public static void setWorkFactor(Configuration conf, int workFactor) {
+    conf.setInt("bzip2.compress.workfactor", workFactor);
+  }
+
+  public static int getWorkFactor(Configuration conf) {
+    return conf.getInt("bzip2.compress.workfactor", 
+                       Bzip2Compressor.DEFAULT_WORK_FACTOR);
+  }
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java

@@ -19,7 +19,10 @@ package org.apache.hadoop.io.nativeio;
 
 import java.io.File;
 import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -27,10 +30,13 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
 import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.Shell;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+
 /**
  * JNI wrappers for various native IO-related calls not available in Java.
  * These functions should generally be used alongside a fallback to another
@@ -39,81 +45,341 @@ import org.apache.commons.logging.LogFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class NativeIO {
-  // Flags for open() call from bits/fcntl.h
-  public static final int O_RDONLY   =    00;
-  public static final int O_WRONLY   =    01;
-  public static final int O_RDWR     =    02;
-  public static final int O_CREAT    =  0100;
-  public static final int O_EXCL     =  0200;
-  public static final int O_NOCTTY   =  0400;
-  public static final int O_TRUNC    = 01000;
-  public static final int O_APPEND   = 02000;
-  public static final int O_NONBLOCK = 04000;
-  public static final int O_SYNC   =  010000;
-  public static final int O_ASYNC  =  020000;
-  public static final int O_FSYNC = O_SYNC;
-  public static final int O_NDELAY = O_NONBLOCK;
-
-  // Flags for posix_fadvise() from bits/fcntl.h
-  /* No further special treatment.  */
-  public static final int POSIX_FADV_NORMAL = 0; 
-  /* Expect random page references.  */
-  public static final int POSIX_FADV_RANDOM = 1; 
-  /* Expect sequential page references.  */
-  public static final int POSIX_FADV_SEQUENTIAL = 2; 
-  /* Will need these pages.  */
-  public static final int POSIX_FADV_WILLNEED = 3; 
-  /* Don't need these pages.  */
-  public static final int POSIX_FADV_DONTNEED = 4; 
-  /* Data will be accessed once.  */
-  public static final int POSIX_FADV_NOREUSE = 5; 
-
-
-  /* Wait upon writeout of all pages
-     in the range before performing the
-     write.  */
-  public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
-  /* Initiate writeout of all those
-     dirty pages in the range which are
-     not presently under writeback.  */
-  public static final int SYNC_FILE_RANGE_WRITE = 2;
-
-  /* Wait upon writeout of all pages in
-     the range after performing the
-     write.  */
-  public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
+  public static class POSIX {
+    // Flags for open() call from bits/fcntl.h
+    public static final int O_RDONLY   =    00;
+    public static final int O_WRONLY   =    01;
+    public static final int O_RDWR     =    02;
+    public static final int O_CREAT    =  0100;
+    public static final int O_EXCL     =  0200;
+    public static final int O_NOCTTY   =  0400;
+    public static final int O_TRUNC    = 01000;
+    public static final int O_APPEND   = 02000;
+    public static final int O_NONBLOCK = 04000;
+    public static final int O_SYNC   =  010000;
+    public static final int O_ASYNC  =  020000;
+    public static final int O_FSYNC = O_SYNC;
+    public static final int O_NDELAY = O_NONBLOCK;
+
+    // Flags for posix_fadvise() from bits/fcntl.h
+    /* No further special treatment.  */
+    public static final int POSIX_FADV_NORMAL = 0;
+    /* Expect random page references.  */
+    public static final int POSIX_FADV_RANDOM = 1;
+    /* Expect sequential page references.  */
+    public static final int POSIX_FADV_SEQUENTIAL = 2;
+    /* Will need these pages.  */
+    public static final int POSIX_FADV_WILLNEED = 3;
+    /* Don't need these pages.  */
+    public static final int POSIX_FADV_DONTNEED = 4;
+    /* Data will be accessed once.  */
+    public static final int POSIX_FADV_NOREUSE = 5;
+
+
+    /* Wait upon writeout of all pages
+       in the range before performing the
+       write.  */
+    public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
+    /* Initiate writeout of all those
+       dirty pages in the range which are
+       not presently under writeback.  */
+    public static final int SYNC_FILE_RANGE_WRITE = 2;
+
+    /* Wait upon writeout of all pages in
+       the range after performing the
+       write.  */
+    public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
+
+    private static final Log LOG = LogFactory.getLog(NativeIO.class);
+
+    private static boolean nativeLoaded = false;
+    private static boolean fadvisePossible = true;
+    private static boolean syncFileRangePossible = true;
+
+    static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
+      "hadoop.workaround.non.threadsafe.getpwuid";
+    static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false;
+
+    private static long cacheTimeout = -1;
+
+    static {
+      if (NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          Configuration conf = new Configuration();
+          workaroundNonThreadSafePasswdCalls = conf.getBoolean(
+            WORKAROUND_NON_THREADSAFE_CALLS_KEY,
+            WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);
+
+          initNative();
+          nativeLoaded = true;
+
+          cacheTimeout = conf.getLong(
+            CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,
+            CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) *
+            1000;
+          LOG.debug("Initialized cache for IDs to User/Group mapping with a " +
+            " cache timeout of " + cacheTimeout/1000 + " seconds.");
+
+        } catch (Throwable t) {
+          // This can happen if the user has an older version of libhadoop.so
+          // installed - in this case we can continue without native IO
+          // after warning
+          LOG.error("Unable to initialize NativeIO libraries", t);
+        }
+      }
+    }
 
-  private static final Log LOG = LogFactory.getLog(NativeIO.class);
+    /**
+     * Return true if the JNI-based native IO extensions are available.
+     */
+    public static boolean isAvailable() {
+      return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
+    }
+
+    /** Wrapper around open(2) */
+    public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
+    /** Wrapper around fstat(2) */
+    private static native Stat fstat(FileDescriptor fd) throws IOException;
+
+    /** Native chmod implementation. On UNIX, it is a wrapper around chmod(2) */
+    private static native void chmodImpl(String path, int mode) throws IOException;
+
+    public static void chmod(String path, int mode) throws IOException {
+      if (!Shell.WINDOWS) {
+        chmodImpl(path, mode);
+      } else {
+        try {
+          chmodImpl(path, mode);
+        } catch (NativeIOException nioe) {
+          if (nioe.getErrorCode() == 3) {
+            throw new NativeIOException("No such file or directory",
+                Errno.ENOENT);
+          } else {
+            LOG.warn(String.format("NativeIO.chmod error (%d): %s",
+                nioe.getErrorCode(), nioe.getMessage()));
+            throw new NativeIOException("Unknown error", Errno.UNKNOWN);
+          }
+        }
+      }
+    }
+
+    /** Wrapper around posix_fadvise(2) */
+    static native void posix_fadvise(
+      FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;
+
+    /** Wrapper around sync_file_range(2) */
+    static native void sync_file_range(
+      FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException;
+
+    /**
+     * Call posix_fadvise on the given file descriptor. See the manpage
+     * for this syscall for more information. On systems where this
+     * call is not available, does nothing.
+     *
+     * @throws NativeIOException if there is an error with the syscall
+     */
+    public static void posixFadviseIfPossible(
+        FileDescriptor fd, long offset, long len, int flags)
+        throws NativeIOException {
+      if (nativeLoaded && fadvisePossible) {
+        try {
+          posix_fadvise(fd, offset, len, flags);
+        } catch (UnsupportedOperationException uoe) {
+          fadvisePossible = false;
+        } catch (UnsatisfiedLinkError ule) {
+          fadvisePossible = false;
+        }
+      }
+    }
+
+    /**
+     * Call sync_file_range on the given file descriptor. See the manpage
+     * for this syscall for more information. On systems where this
+     * call is not available, does nothing.
+     *
+     * @throws NativeIOException if there is an error with the syscall
+     */
+    public static void syncFileRangeIfPossible(
+        FileDescriptor fd, long offset, long nbytes, int flags)
+        throws NativeIOException {
+      if (nativeLoaded && syncFileRangePossible) {
+        try {
+          sync_file_range(fd, offset, nbytes, flags);
+        } catch (UnsupportedOperationException uoe) {
+          syncFileRangePossible = false;
+        } catch (UnsatisfiedLinkError ule) {
+          syncFileRangePossible = false;
+        }
+      }
+    }
+
+    /** Linux only methods used for getOwner() implementation */
+    private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException;
+    private static native String getUserName(long uid) throws IOException;
+
+    /**
+     * Result type of the fstat call
+     */
+    public static class Stat {
+      private int ownerId, groupId;
+      private String owner, group;
+      private int mode;
+
+      // Mode constants
+      public static final int S_IFMT = 0170000;      /* type of file */
+      public static final int   S_IFIFO  = 0010000;  /* named pipe (fifo) */
+      public static final int   S_IFCHR  = 0020000;  /* character special */
+      public static final int   S_IFDIR  = 0040000;  /* directory */
+      public static final int   S_IFBLK  = 0060000;  /* block special */
+      public static final int   S_IFREG  = 0100000;  /* regular */
+      public static final int   S_IFLNK  = 0120000;  /* symbolic link */
+      public static final int   S_IFSOCK = 0140000;  /* socket */
+      public static final int   S_IFWHT  = 0160000;  /* whiteout */
+      public static final int S_ISUID = 0004000;  /* set user id on execution */
+      public static final int S_ISGID = 0002000;  /* set group id on execution */
+      public static final int S_ISVTX = 0001000;  /* save swapped text even after use */
+      public static final int S_IRUSR = 0000400;  /* read permission, owner */
+      public static final int S_IWUSR = 0000200;  /* write permission, owner */
+      public static final int S_IXUSR = 0000100;  /* execute/search permission, owner */
+
+      Stat(int ownerId, int groupId, int mode) {
+        this.ownerId = ownerId;
+        this.groupId = groupId;
+        this.mode = mode;
+      }
+
+      @Override
+      public String toString() {
+        return "Stat(owner='" + owner + "', group='" + group + "'" +
+          ", mode=" + mode + ")";
+      }
+
+      public String getOwner() {
+        return owner;
+      }
+      public String getGroup() {
+        return group;
+      }
+      public int getMode() {
+        return mode;
+      }
+    }
+
+    /**
+     * Returns the file stat for a file descriptor.
+     *
+     * @param fd file descriptor.
+     * @return the file descriptor file stat.
+     * @throws IOException thrown if there was an IO error while obtaining the file stat.
+     */
+    public static Stat getFstat(FileDescriptor fd) throws IOException {
+      Stat stat = fstat(fd);
+      stat.owner = getName(IdCache.USER, stat.ownerId);
+      stat.group = getName(IdCache.GROUP, stat.groupId);
+      return stat;
+    }
+
+    private static String getName(IdCache domain, int id) throws IOException {
+      Map<Integer, CachedName> idNameCache = (domain == IdCache.USER)
+        ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE;
+      String name;
+      CachedName cachedName = idNameCache.get(id);
+      long now = System.currentTimeMillis();
+      if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) {
+        name = cachedName.name;
+      } else {
+        name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id);
+        if (LOG.isDebugEnabled()) {
+          String type = (domain == IdCache.USER) ? "UserName" : "GroupName";
+          LOG.debug("Got " + type + " " + name + " for ID " + id +
+            " from the native implementation");
+        }
+        cachedName = new CachedName(name, now);
+        idNameCache.put(id, cachedName);
+      }
+      return name;
+    }
+
+    static native String getUserName(int uid) throws IOException;
+    static native String getGroupName(int uid) throws IOException;
+
+    private static class CachedName {
+      final long timestamp;
+      final String name;
+
+      public CachedName(String name, long timestamp) {
+        this.name = name;
+        this.timestamp = timestamp;
+      }
+    }
+
+    private static final Map<Integer, CachedName> USER_ID_NAME_CACHE =
+      new ConcurrentHashMap<Integer, CachedName>();
+
+    private static final Map<Integer, CachedName> GROUP_ID_NAME_CACHE =
+      new ConcurrentHashMap<Integer, CachedName>();
+
+    private enum IdCache { USER, GROUP }
+  }
 
-  private static boolean nativeLoaded = false;
   private static boolean workaroundNonThreadSafePasswdCalls = false;
-  private static boolean fadvisePossible = true;
-  private static boolean syncFileRangePossible = true;
 
-  static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
-    "hadoop.workaround.non.threadsafe.getpwuid";
-  static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false;
 
-  private static long cacheTimeout = -1;
+  public static class Windows {
+    // Flags for CreateFile() call on Windows
+    public static final long GENERIC_READ = 0x80000000L;
+    public static final long GENERIC_WRITE = 0x40000000L;
+
+    public static final long FILE_SHARE_READ = 0x00000001L;
+    public static final long FILE_SHARE_WRITE = 0x00000002L;
+    public static final long FILE_SHARE_DELETE = 0x00000004L;
+
+    public static final long CREATE_NEW = 1;
+    public static final long CREATE_ALWAYS = 2;
+    public static final long OPEN_EXISTING = 3;
+    public static final long OPEN_ALWAYS = 4;
+    public static final long TRUNCATE_EXISTING = 5;
+
+    public static final long FILE_BEGIN = 0;
+    public static final long FILE_CURRENT = 1;
+    public static final long FILE_END = 2;
+
+    /** Wrapper around CreateFile() on Windows */
+    public static native FileDescriptor createFile(String path,
+        long desiredAccess, long shareMode, long creationDisposition)
+        throws IOException;
+
+    /** Wrapper around SetFilePointer() on Windows */
+    public static native long setFilePointer(FileDescriptor fd,
+        long distanceToMove, long moveMethod) throws IOException;
+
+    /** Windows only methods used for getOwner() implementation */
+    private static native String getOwner(FileDescriptor fd) throws IOException;
+
+    static {
+      if (NativeCodeLoader.isNativeCodeLoaded()) {
+        try {
+          initNative();
+          nativeLoaded = true;
+        } catch (Throwable t) {
+          // This can happen if the user has an older version of libhadoop.so
+          // installed - in this case we can continue without native IO
+          // after warning
+          LOG.error("Unable to initialize NativeIO libraries", t);
+        }
+      }
+    }
+  }
+
+  private static final Log LOG = LogFactory.getLog(NativeIO.class);
+
+  private static boolean nativeLoaded = false;
 
   static {
     if (NativeCodeLoader.isNativeCodeLoaded()) {
       try {
-        Configuration conf = new Configuration();
-        workaroundNonThreadSafePasswdCalls = conf.getBoolean(
-          WORKAROUND_NON_THREADSAFE_CALLS_KEY,
-          WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT);
-
         initNative();
         nativeLoaded = true;
-
-        cacheTimeout = conf.getLong(
-          CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY,
-          CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) *
-          1000;
-        LOG.debug("Initialized cache for IDs to User/Group mapping with a" +
-          " cache timeout of " + cacheTimeout/1000 + " seconds.");
-
       } catch (Throwable t) {
         // This can happen if the user has an older version of libhadoop.so
         // installed - in this case we can continue without native IO
@@ -130,169 +396,161 @@ public class NativeIO {
     return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
   }
 
-  /** Wrapper around open(2) */
-  public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
-  /** Wrapper around fstat(2) */
-  private static native Stat fstat(FileDescriptor fd) throws IOException;
-  /** Wrapper around chmod(2) */
-  public static native void chmod(String path, int mode) throws IOException;
-
-  /** Wrapper around posix_fadvise(2) */
-  static native void posix_fadvise(
-    FileDescriptor fd, long offset, long len, int flags) throws NativeIOException;
-
-  /** Wrapper around sync_file_range(2) */
-  static native void sync_file_range(
-    FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException;
-
   /** Initialize the JNI method ID and class ID cache */
   private static native void initNative();
 
-  /**
-   * Call posix_fadvise on the given file descriptor. See the manpage
-   * for this syscall for more information. On systems where this
-   * call is not available, does nothing.
-   *
-   * @throws NativeIOException if there is an error with the syscall
-   */
-  public static void posixFadviseIfPossible(
-      FileDescriptor fd, long offset, long len, int flags)
-      throws NativeIOException {
-    if (nativeLoaded && fadvisePossible) {
-      try {
-        posix_fadvise(fd, offset, len, flags);
-      } catch (UnsupportedOperationException uoe) {
-        fadvisePossible = false;
-      } catch (UnsatisfiedLinkError ule) {
-        fadvisePossible = false;
-      }
+  private static class CachedUid {
+    final long timestamp;
+    final String username;
+    public CachedUid(String username, long timestamp) {
+      this.timestamp = timestamp;
+      this.username = username;
     }
   }
-
-  /**
-   * Call sync_file_range on the given file descriptor. See the manpage
-   * for this syscall for more information. On systems where this
-   * call is not available, does nothing.
-   *
-   * @throws NativeIOException if there is an error with the syscall
-   */
-  public static void syncFileRangeIfPossible(
-      FileDescriptor fd, long offset, long nbytes, int flags)
-      throws NativeIOException {
-    if (nativeLoaded && syncFileRangePossible) {
-      try {
-        sync_file_range(fd, offset, nbytes, flags);
-      } catch (UnsupportedOperationException uoe) {
-        syncFileRangePossible = false;
-      } catch (UnsatisfiedLinkError ule) {
-        syncFileRangePossible = false;
+  private static final Map<Long, CachedUid> uidCache =
+      new ConcurrentHashMap<Long, CachedUid>();
+  private static long cacheTimeout;
+  private static boolean initialized = false;
+
+  public static String getOwner(FileDescriptor fd) throws IOException {
+    ensureInitialized();
+    if (Shell.WINDOWS) {
+      String owner = Windows.getOwner(fd);
+      int i = owner.indexOf('\\');
+      if (i != -1)
+        owner = owner.substring(i + 1);
+      return owner;
+    } else {
+      long uid = POSIX.getUIDforFDOwnerforOwner(fd);
+      CachedUid cUid = uidCache.get(uid);
+      long now = System.currentTimeMillis();
+      if (cUid != null && (cUid.timestamp + cacheTimeout) > now) {
+        return cUid.username;
       }
+      String user = POSIX.getUserName(uid);
+      LOG.info("Got UserName " + user + " for UID " + uid
+          + " from the native implementation");
+      cUid = new CachedUid(user, now);
+      uidCache.put(uid, cUid);
+      return user;
     }
   }
 
   /**
-   * Result type of the fstat call
+   * Create a FileInputStream that shares delete permission on the
+   * file opened, i.e. other process can delete the file the
+   * FileInputStream is reading. Only Windows implementation uses
+   * the native interface.
    */
-  public static class Stat {
-    private int ownerId, groupId;
-    private String owner, group;
-    private int mode;
-
-    // Mode constants
-    public static final int S_IFMT = 0170000;      /* type of file */
-    public static final int   S_IFIFO  = 0010000;  /* named pipe (fifo) */
-    public static final int   S_IFCHR  = 0020000;  /* character special */
-    public static final int   S_IFDIR  = 0040000;  /* directory */
-    public static final int   S_IFBLK  = 0060000;  /* block special */
-    public static final int   S_IFREG  = 0100000;  /* regular */
-    public static final int   S_IFLNK  = 0120000;  /* symbolic link */
-    public static final int   S_IFSOCK = 0140000;  /* socket */
-    public static final int   S_IFWHT  = 0160000;  /* whiteout */
-    public static final int S_ISUID = 0004000;  /* set user id on execution */
-    public static final int S_ISGID = 0002000;  /* set group id on execution */
-    public static final int S_ISVTX = 0001000;  /* save swapped text even after use */
-    public static final int S_IRUSR = 0000400;  /* read permission, owner */
-    public static final int S_IWUSR = 0000200;  /* write permission, owner */
-    public static final int S_IXUSR = 0000100;  /* execute/search permission, owner */
-
-    Stat(int ownerId, int groupId, int mode) {
-      this.ownerId = ownerId;
-      this.groupId = groupId;
-      this.mode = mode;
-    }
-
-    @Override
-    public String toString() {
-      return "Stat(owner='" + owner + "', group='" + group + "'" +
-        ", mode=" + mode + ")";
-    }
-
-    public String getOwner() {
-      return owner;
-    }
-    public String getGroup() {
-      return group;
-    }
-    public int getMode() {
-      return mode;
+  public static FileInputStream getShareDeleteFileInputStream(File f)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      // On Linux the default FileInputStream shares delete permission
+      // on the file opened.
+      //
+      return new FileInputStream(f);
+    } else {
+      // Use Windows native interface to create a FileInputStream that
+      // shares delete permission on the file opened.
+      //
+      FileDescriptor fd = Windows.createFile(
+          f.getAbsolutePath(),
+          Windows.GENERIC_READ,
+          Windows.FILE_SHARE_READ |
+              Windows.FILE_SHARE_WRITE |
+              Windows.FILE_SHARE_DELETE,
+          Windows.OPEN_EXISTING);
+      return new FileInputStream(fd);
     }
   }
 
-  static native String getUserName(int uid) throws IOException;
-
-  static native String getGroupName(int uid) throws IOException;
-
-  private static class CachedName {
-    final long timestamp;
-    final String name;
-
-    public CachedName(String name, long timestamp) {
-      this.name = name;
-      this.timestamp = timestamp;
+  /**
+   * Create a FileInputStream that shares delete permission on the
+   * file opened at a given offset, i.e. other process can delete
+   * the file the FileInputStream is reading. Only Windows implementation
+   * uses the native interface.
+   */
+  public static FileInputStream getShareDeleteFileInputStream(File f, long seekOffset)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      RandomAccessFile rf = new RandomAccessFile(f, "r");
+      if (seekOffset > 0) {
+        rf.seek(seekOffset);
+      }
+      return new FileInputStream(rf.getFD());
+    } else {
+      // Use Windows native interface to create a FileInputStream that
+      // shares delete permission on the file opened, and set it to the
+      // given offset.
+      //
+      FileDescriptor fd = NativeIO.Windows.createFile(
+          f.getAbsolutePath(),
+          NativeIO.Windows.GENERIC_READ,
+          NativeIO.Windows.FILE_SHARE_READ |
+              NativeIO.Windows.FILE_SHARE_WRITE |
+              NativeIO.Windows.FILE_SHARE_DELETE,
+          NativeIO.Windows.OPEN_EXISTING);
+      if (seekOffset > 0)
+        NativeIO.Windows.setFilePointer(fd, seekOffset, NativeIO.Windows.FILE_BEGIN);
+      return new FileInputStream(fd);
     }
   }
 
-  private static final Map<Integer, CachedName> USER_ID_NAME_CACHE =
-    new ConcurrentHashMap<Integer, CachedName>();
-
-  private static final Map<Integer, CachedName> GROUP_ID_NAME_CACHE =
-    new ConcurrentHashMap<Integer, CachedName>();
-
-  private enum IdCache { USER, GROUP }
-
-  private static String getName(IdCache domain, int id) throws IOException {
-    Map<Integer, CachedName> idNameCache = (domain == IdCache.USER)
-      ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE;
-    String name;
-    CachedName cachedName = idNameCache.get(id);
-    long now = System.currentTimeMillis();
-    if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) {
-      name = cachedName.name;
+  /**
+   * Create the specified File for write access, ensuring that it does not exist.
+   * @param f the file that we want to create
+   * @param permissions we want to have on the file (if security is enabled)
+   *
+   * @throws AlreadyExistsException if the file already exists
+   * @throws IOException if any other error occurred
+   */
+  public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions)
+      throws IOException {
+    if (!Shell.WINDOWS) {
+      // Use the native wrapper around open(2)
+      try {
+        FileDescriptor fd = NativeIO.POSIX.open(f.getAbsolutePath(),
+            NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT
+                | NativeIO.POSIX.O_EXCL, permissions);
+        return new FileOutputStream(fd);
+      } catch (NativeIOException nioe) {
+        if (nioe.getErrno() == Errno.EEXIST) {
+          throw new AlreadyExistsException(nioe);
+        }
+        throw nioe;
+      }
     } else {
-      name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id);
-      if (LOG.isDebugEnabled()) {
-        String type = (domain == IdCache.USER) ? "UserName" : "GroupName";
-        LOG.debug("Got " + type + " " + name + " for ID " + id +
-          " from the native implementation");
+      // Use the Windows native APIs to create equivalent FileOutputStream
+      try {
+        FileDescriptor fd = NativeIO.Windows.createFile(f.getCanonicalPath(),
+            NativeIO.Windows.GENERIC_WRITE,
+            NativeIO.Windows.FILE_SHARE_DELETE
+                | NativeIO.Windows.FILE_SHARE_READ
+                | NativeIO.Windows.FILE_SHARE_WRITE,
+            NativeIO.Windows.CREATE_NEW);
+        NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions);
+        return new FileOutputStream(fd);
+      } catch (NativeIOException nioe) {
+        if (nioe.getErrorCode() == 80) {
+          // ERROR_FILE_EXISTS
+          // 80 (0x50)
+          // The file exists
+          throw new AlreadyExistsException(nioe);
+        }
+        throw nioe;
       }
-      cachedName = new CachedName(name, now);
-      idNameCache.put(id, cachedName);
     }
-    return name;
   }
 
-  /**
-   * Returns the file stat for a file descriptor.
-   *
-   * @param fd file descriptor.
-   * @return the file descriptor file stat.
-   * @throws IOException thrown if there was an IO error while obtaining the file stat.
-   */
-  public static Stat getFstat(FileDescriptor fd) throws IOException {
-    Stat stat = fstat(fd);
-    stat.owner = getName(IdCache.USER, stat.ownerId);
-    stat.group = getName(IdCache.GROUP, stat.groupId);
-    return stat;
+  private synchronized static void ensureInitialized() {
+    if (!initialized) {
+      cacheTimeout =
+          new Configuration().getLong("hadoop.security.uid.cache.secs",
+              4*60*60) * 1000;
+      LOG.info("Initialized cache for UID to User mapping with a cache" +
+          " timeout of " + cacheTimeout/1000 + " seconds.");
+      initialized = true;
+    }
   }
   
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java

@@ -18,20 +18,40 @@
 package org.apache.hadoop.io.nativeio;
 
 import java.io.IOException;
+import org.apache.hadoop.util.Shell;
+
 
 /**
  * An exception generated by a call to the native IO code.
  *
- * These exceptions simply wrap <i>errno</i> result codes.
+ * These exceptions simply wrap <i>errno</i> result codes on Linux,
+ * or the System Error Code on Windows.
  */
 public class NativeIOException extends IOException {
   private static final long serialVersionUID = 1L;
 
   private Errno errno;
 
+  // Java has no unsigned primitive error code. Use a signed 32-bit
+  // integer to hold the unsigned 32-bit integer.
+  private int errorCode;
+
   public NativeIOException(String msg, Errno errno) {
     super(msg);
     this.errno = errno;
+    // Windows error code is always set to ERROR_SUCCESS on Linux,
+    // i.e. no failure on Windows
+    this.errorCode = 0;
+  }
+
+  public NativeIOException(String msg, int errorCode) {
+    super(msg);
+    this.errorCode = errorCode;
+    this.errno = Errno.UNKNOWN;
+  }
+
+  public long getErrorCode() {
+    return errorCode;
   }
 
   public Errno getErrno() {
@@ -40,8 +60,10 @@ public class NativeIOException extends IOException {
 
   @Override
   public String toString() {
-    return errno.toString() + ": " + super.getMessage();
+    if (Shell.WINDOWS)
+      return errorCode + ": " + super.getMessage();
+    else
+      return errno.toString() + ": " + super.getMessage();
   }
 }
 
-

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.ipc;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.net.InetSocketAddress;
@@ -39,7 +40,7 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.ipc.RPC.RpcInvoker;
-import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestProto;
+import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -128,25 +129,12 @@ public class ProtobufRpcEngine implements RpcEngine {
           .getProtocolVersion(protocol);
     }
 
-    private RequestProto constructRpcRequest(Method method,
-        Object[] params) throws ServiceException {
-      RequestProto rpcRequest;
-      RequestProto.Builder builder = RequestProto
+    private RequestHeaderProto constructRpcRequestHeader(Method method) {
+      RequestHeaderProto.Builder builder = RequestHeaderProto
           .newBuilder();
       builder.setMethodName(method.getName());
+     
 
-      if (params.length != 2) { // RpcController + Message
-        throw new ServiceException("Too many parameters for request. Method: ["
-            + method.getName() + "]" + ", Expected: 2, Actual: "
-            + params.length);
-      }
-      if (params[1] == null) {
-        throw new ServiceException("null param while calling Method: ["
-            + method.getName() + "]");
-      }
-
-      Message param = (Message) params[1];
-      builder.setRequest(param.toByteString());
       // For protobuf, {@code protocol} used when creating client side proxy is
       // the interface extending BlockingInterface, which has the annotations 
       // such as ProtocolName etc.
@@ -160,8 +148,7 @@ public class ProtobufRpcEngine implements RpcEngine {
       // For PB this may limit the use of mixins on client side.
       builder.setDeclaringClassProtocolName(protocolName);
       builder.setClientProtocolVersion(clientProtocolVersion);
-      rpcRequest = builder.build();
-      return rpcRequest;
+      return builder.build();
     }
 
     /**
@@ -189,8 +176,18 @@ public class ProtobufRpcEngine implements RpcEngine {
       if (LOG.isDebugEnabled()) {
         startTime = Time.now();
       }
+      
+      if (args.length != 2) { // RpcController + Message
+        throw new ServiceException("Too many parameters for request. Method: ["
+            + method.getName() + "]" + ", Expected: 2, Actual: "
+            + args.length);
+      }
+      if (args[1] == null) {
+        throw new ServiceException("null param while calling Method: ["
+            + method.getName() + "]");
+      }
 
-      RequestProto rpcRequest = constructRpcRequest(method, args);
+      RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method);
       RpcResponseWrapper val = null;
       
       if (LOG.isTraceEnabled()) {
@@ -198,9 +195,12 @@ public class ProtobufRpcEngine implements RpcEngine {
             remoteId + ": " + method.getName() +
             " {" + TextFormat.shortDebugString((Message) args[1]) + "}");
       }
+
+
+      Message theRequest = (Message) args[1];
       try {
         val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
-            new RpcRequestWrapper(rpcRequest), remoteId);
+            new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId);
 
       } catch (Throwable e) {
         if (LOG.isTraceEnabled()) {
@@ -275,20 +275,25 @@ public class ProtobufRpcEngine implements RpcEngine {
    * use type Writable as a wrapper to work across multiple RpcEngine kinds.
    */
   private static class RpcRequestWrapper implements Writable {
-    RequestProto message;
+    RequestHeaderProto requestHeader;
+    Message theRequest; // for clientSide, the request is here
+    byte[] theRequestRead; // for server side, the request is here
 
     @SuppressWarnings("unused")
     public RpcRequestWrapper() {
     }
 
-    RpcRequestWrapper(RequestProto message) {
-      this.message = message;
+    RpcRequestWrapper(RequestHeaderProto requestHeader, Message theRequest) {
+      this.requestHeader = requestHeader;
+      this.theRequest = theRequest;
     }
 
     @Override
     public void write(DataOutput out) throws IOException {
-      ((Message)message).writeDelimitedTo(
-          DataOutputOutputStream.constructOutputStream(out));
+      OutputStream os = DataOutputOutputStream.constructOutputStream(out);
+      
+      ((Message)requestHeader).writeDelimitedTo(os);
+      theRequest.writeDelimitedTo(os);
     }
 
     @Override
@@ -296,13 +301,16 @@ public class ProtobufRpcEngine implements RpcEngine {
       int length = ProtoUtil.readRawVarint32(in);
       byte[] bytes = new byte[length];
       in.readFully(bytes);
-      message = RequestProto.parseFrom(bytes);
+      requestHeader = RequestHeaderProto.parseFrom(bytes);
+      length = ProtoUtil.readRawVarint32(in);
+      theRequestRead = new byte[length];
+      in.readFully(theRequestRead);
     }
     
     @Override
     public String toString() {
-      return message.getDeclaringClassProtocolName() + "." +
-          message.getMethodName();
+      return requestHeader.getDeclaringClassProtocolName() + "." +
+          requestHeader.getMethodName();
     }
   }
 
@@ -434,7 +442,7 @@ public class ProtobufRpcEngine implements RpcEngine {
       public Writable call(RPC.Server server, String connectionProtocolName,
           Writable writableRequest, long receiveTime) throws Exception {
         RpcRequestWrapper request = (RpcRequestWrapper) writableRequest;
-        RequestProto rpcRequest = request.message;
+        RequestHeaderProto rpcRequest = request.requestHeader;
         String methodName = rpcRequest.getMethodName();
         
         
@@ -474,7 +482,8 @@ public class ProtobufRpcEngine implements RpcEngine {
         }
         Message prototype = service.getRequestPrototype(methodDescriptor);
         Message param = prototype.newBuilderForType()
-            .mergeFrom(rpcRequest.getRequest()).build();
+            .mergeFrom(request.theRequestRead).build();
+        
         Message result;
         try {
           long startTime = Time.now();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -24,6 +24,7 @@ import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
 import java.net.BindException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -313,6 +314,14 @@ public abstract class Server {
     return (addr == null) ? null : addr.getHostAddress();
   }
 
+  /** Returns the RPC remote user when invoked inside an RPC.  Note this
+   *  may be different than the current user if called within another doAs
+   *  @return connection's UGI or null if not an RPC
+   */
+  public static UserGroupInformation getRemoteUser() {
+    Call call = CurCall.get();
+    return (call != null) ? call.connection.user : null;
+  }
  
   /** Return true if the invocation was through an RPC.
    */
@@ -1553,9 +1562,6 @@ public abstract class Server {
       UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext);
       if (saslServer == null) {
         user = protocolUser;
-        if (user != null) {
-          user.setAuthenticationMethod(AuthMethod.SIMPLE);
-        }
       } else {
         // user is authenticated
         user.setAuthenticationMethod(authMethod);
@@ -1783,6 +1789,9 @@ public abstract class Server {
                   );
             }
           } catch (Throwable e) {
+            if (e instanceof UndeclaredThrowableException) {
+              e = e.getCause();
+            }
             String logMsg = getName() + ", call " + call + ": error: " + e;
             if (e instanceof RuntimeException || e instanceof Error) {
               // These exception types indicate something is probably wrong

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java

@@ -140,10 +140,12 @@ public class MetricsServlet extends HttpServlet {
    */
   void printMap(PrintWriter out, Map<String, Map<String, List<TagsMetricsPair>>> map) {
     for (Map.Entry<String, Map<String, List<TagsMetricsPair>>> context : map.entrySet()) {
-      out.println(context.getKey());
+      out.print(context.getKey());
+      out.print("\n");
       for (Map.Entry<String, List<TagsMetricsPair>> record : context.getValue().entrySet()) {
         indent(out, 1);
-        out.println(record.getKey());
+        out.print(record.getKey());
+        out.print("\n");
         for (TagsMetricsPair pair : record.getValue()) {
           indent(out, 2);
           // Prints tag values in the form "{key=value,key=value}:"
@@ -159,7 +161,7 @@ public class MetricsServlet extends HttpServlet {
             out.print("=");
             out.print(tagValue.getValue().toString());
           }
-          out.println("}:");
+          out.print("}:\n");
           
           // Now print metric values, one per line
           for (Map.Entry<String, Number> metricValue : 
@@ -167,7 +169,8 @@ public class MetricsServlet extends HttpServlet {
             indent(out, 3);
             out.print(metricValue.getKey());
             out.print("=");
-            out.println(metricValue.getValue().toString());
+            out.print(metricValue.getValue().toString());
+            out.print("\n");
           }
         }
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java

@@ -192,7 +192,7 @@ public class Buffer implements Comparable, Cloneable {
     int hash = 1;
     byte[] b = this.get();
     for (int i = 0; i < count; i++)
-      hash = (31 * hash) + (int)b[i];
+      hash = (31 * hash) + b[i];
     return hash;
   }
   

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java

@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * Interface that alll the serializers have to implement.
+ * Interface that all the serializers have to implement.
  * 
  * @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
  */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java

@@ -30,6 +30,7 @@ import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.ServiceLoader;
 import java.util.Set;
 
@@ -219,7 +220,7 @@ public class SecurityUtil {
     if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) {
       fqdn = getLocalHostName();
     }
-    return components[0] + "/" + fqdn.toLowerCase() + "@" + components[2];
+    return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2];
   }
   
   static String getLocalHostName() throws UnknownHostException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java

@@ -86,7 +86,8 @@ public class ShellBasedUnixGroupsMapping
       LOG.warn("got exception trying to get groups for user " + user, e);
     }
     
-    StringTokenizer tokenizer = new StringTokenizer(result);
+    StringTokenizer tokenizer =
+        new StringTokenizer(result, Shell.TOKEN_SEPARATOR_REGEX);
     List<String> groups = new LinkedList<String>();
     while (tokenizer.hasMoreTokens()) {
       groups.add(tokenizer.nextToken());

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -67,6 +67,8 @@ import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Time;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * User and group information for Hadoop.
  * This class wraps around a JAAS Subject and provides methods to determine the
@@ -713,7 +715,8 @@ public class UserGroupInformation {
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
-  synchronized static void setLoginUser(UserGroupInformation ugi) {
+  @VisibleForTesting
+  public synchronized static void setLoginUser(UserGroupInformation ugi) {
     // if this is to become stable, should probably logout the currently
     // logged in ugi if it's different
     loginUser = ugi;
@@ -1498,7 +1501,7 @@ public class UserGroupInformation {
       } else if (cause instanceof InterruptedException) {
         throw (InterruptedException) cause;
       } else {
-        throw new UndeclaredThrowableException(pae,"Unknown exception in doAs");
+        throw new UndeclaredThrowableException(cause);
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.Enumeration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class ClassUtil {
+  /** 
+   * Find a jar that contains a class of the same name, if any.
+   * It will return a jar file, even if that is not the first thing
+   * on the class path that has a class with the same name.
+   * 
+   * @param clazz the class to find.
+   * @return a jar file that contains the class, or null.
+   * @throws IOException
+   */
+  public static String findContainingJar(Class clazz) {
+    ClassLoader loader = clazz.getClassLoader();
+    String classFile = clazz.getName().replaceAll("\\.", "/") + ".class";
+    try {
+      for (Enumeration itr = loader.getResources(classFile);
+          itr.hasMoreElements();) {
+        URL url = (URL) itr.nextElement();
+        if ("jar".equals(url.getProtocol())) {
+          String toReturn = url.getPath();
+          if (toReturn.startsWith("file:")) {
+            toReturn = toReturn.substring("file:".length());
+          }
+          toReturn = URLDecoder.decode(toReturn, "UTF-8");
+          return toReturn.replaceAll("!.*$", "");
+        }
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return null;
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java

@@ -60,7 +60,7 @@ class NativeCrc32 {
         fileName, basePos);
   }
   
-  private static native void nativeVerifyChunkedSums(
+    private static native void nativeVerifyChunkedSums(
       int bytesPerSum, int checksumType,
       ByteBuffer sums, int sumsOffset,
       ByteBuffer data, int dataOffset, int dataLength,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java

@@ -32,9 +32,10 @@ public class PlatformName {
    * The complete platform 'name' to identify the platform as 
    * per the java-vm.
    */
-  private static final String platformName = System.getProperty("os.name") + "-" + 
-    System.getProperty("os.arch") + "-" +
-    System.getProperty("sun.arch.data.model");
+  private static final String platformName =
+      (Shell.WINDOWS ? System.getenv("os") : System.getProperty("os.name"))
+      + "-" + System.getProperty("os.arch")
+      + "-" + System.getProperty("sun.arch.data.model");
   
   /**
    * Get the complete platform as per the java-vm.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

@@ -21,6 +21,7 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
@@ -44,46 +45,208 @@ abstract public class Shell {
   
   public static final Log LOG = LogFactory.getLog(Shell.class);
   
+  private static boolean IS_JAVA7_OR_ABOVE =
+      System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0;
+
+  public static boolean isJava7OrAbove() {
+    return IS_JAVA7_OR_ABOVE;
+  }
+
   /** a Unix command to get the current user's name */
   public final static String USER_NAME_COMMAND = "whoami";
+
+  /** Windows CreateProcess synchronization object */
+  public static final Object WindowsProcessLaunchLock = new Object();
+
   /** a Unix command to get the current user's groups list */
   public static String[] getGroupsCommand() {
-    return new String[]{"bash", "-c", "groups"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
+                    : new String[]{"bash", "-c", "groups"};
   }
+
   /** a Unix command to get a given user's groups list */
   public static String[] getGroupsForUserCommand(final String user) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "id -Gn " + user};
+    return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""}
+                    : new String [] {"bash", "-c", "id -Gn " + user};
   }
+
   /** a Unix command to get a given netgroup's user list */
   public static String[] getUsersForNetgroupCommand(final String netgroup) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "getent netgroup " + netgroup};
+    return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
+                    : new String [] {"bash", "-c", "getent netgroup " + netgroup};
+  }
+
+  /** Return a command to get permission information. */
+  public static String[] getGetPermissionCommand() {
+    return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" }
+                     : new String[] { "/bin/ls", "-ld" };
+  }
+
+  /** Return a command to set permission */
+  public static String[] getSetPermissionCommand(String perm, boolean recursive) {
+    if (recursive) {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
+                         : new String[] { "chmod", "-R", perm };
+    } else {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
+                       : new String[] { "chmod", perm };
+    }
   }
+
+  /**
+   * Return a command to set permission for specific file.
+   * 
+   * @param perm String permission to set
+   * @param recursive boolean true to apply to all sub-directories recursively
+   * @param file String file to set
+   * @return String[] containing command and arguments
+   */
+  public static String[] getSetPermissionCommand(String perm, boolean recursive,
+                                                 String file) {
+    String[] baseCmd = getSetPermissionCommand(perm, recursive);
+    String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
+    cmdWithFile[cmdWithFile.length - 1] = file;
+    return cmdWithFile;
+  }
+
+  /** Return a command to set owner */
+  public static String[] getSetOwnerCommand(String owner) {
+    return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" }
+                     : new String[] { "chown", owner };
+  }
+  
+  /** Return a command to create symbolic links */
+  public static String[] getSymlinkCommand(String target, String link) {
+    return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
+                   : new String[] { "ln", "-s", target, link };
+  }
+
   /** a Unix command to set permission */
   public static final String SET_PERMISSION_COMMAND = "chmod";
   /** a Unix command to set owner */
   public static final String SET_OWNER_COMMAND = "chown";
+
+  /** a Unix command to set the change user's groups list */
   public static final String SET_GROUP_COMMAND = "chgrp";
   /** a Unix command to create a link */
   public static final String LINK_COMMAND = "ln";
   /** a Unix command to get a link target */
   public static final String READ_LINK_COMMAND = "readlink";
-  /** Return a Unix command to get permission information. */
-  public static String[] getGET_PERMISSION_COMMAND() {
-    //force /bin/ls, except on windows.
-    return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"};
-  }
 
   /**Time after which the executing script would be timedout*/
   protected long timeOutInterval = 0L;
   /** If or not script timed out*/
   private AtomicBoolean timedOut;
 
+
+  /** Centralized logic to discover and validate the sanity of the Hadoop 
+   *  home directory. Returns either NULL or a directory that exists and 
+   *  was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV 
+   *  variable.  This does a lot of work so it should only be called 
+   *  privately for initialization once per process.
+   **/
+  private static String checkHadoopHome() {
+
+    // first check the Dflag hadoop.home.dir with JVM scope
+    String home = System.getProperty("hadoop.home.dir");
+
+    // fall back to the system/user-global env variable
+    if (home == null) {
+      home = System.getenv("HADOOP_HOME");
+    }
+
+    try {
+       // couldn't find either setting for hadoop's home directory
+       if (home == null) {
+         throw new IOException("HADOOP_HOME or hadoop.home.dir are not set.");
+       }
+
+       if (home.startsWith("\"") && home.endsWith("\"")) {
+         home = home.substring(1, home.length()-1);
+       }
+
+       // check that the home setting is actually a directory that exists
+       File homedir = new File(home);
+       if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) {
+         throw new IOException("Hadoop home directory " + homedir
+           + " does not exist, is not a directory, or is not an absolute path.");
+       }
+
+       home = homedir.getCanonicalPath();
+
+    } catch (IOException ioe) {
+       LOG.error("Failed to detect a valid hadoop home directory", ioe);
+       home = null;
+    }
+    
+    return home;
+  }
+  private static String HADOOP_HOME_DIR = checkHadoopHome();
+
+  // Public getter, throws an exception if HADOOP_HOME failed validation
+  // checks and is being referenced downstream.
+  public static final String getHadoopHome() throws IOException {
+    if (HADOOP_HOME_DIR == null) {
+      throw new IOException("Misconfigured HADOOP_HOME cannot be referenced.");
+    }
+
+    return HADOOP_HOME_DIR;
+  }
+
+  /** fully qualify the path to a binary that should be in a known hadoop 
+   *  bin location. This is primarily useful for disambiguating call-outs 
+   *  to executable sub-components of Hadoop to avoid clashes with other 
+   *  executables that may be in the path.  Caveat:  this call doesn't 
+   *  just format the path to the bin directory.  It also checks for file 
+   *  existence of the composed path. The output of this call should be 
+   *  cached by callers.
+   * */
+  public static final String getQualifiedBinPath(String executable) 
+  throws IOException {
+    // construct hadoop bin path to the specified executable
+    String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" 
+      + File.separator + executable;
+
+    File exeFile = new File(fullExeName);
+    if (!exeFile.exists()) {
+      throw new IOException("Could not locate executable " + fullExeName
+        + " in the Hadoop binaries.");
+    }
+
+    return exeFile.getCanonicalPath();
+  }
+
   /** Set to true on Windows platforms */
   public static final boolean WINDOWS /* borrowed from Path.WINDOWS */
                 = System.getProperty("os.name").startsWith("Windows");
+
+  public static final boolean LINUX
+                = System.getProperty("os.name").startsWith("Linux");
   
+  /** a Windows utility to emulate Unix commands */
+  public static final String WINUTILS = getWinUtilsPath();
+
+  public static final String getWinUtilsPath() {
+    String winUtilsPath = null;
+
+    try {
+      if (WINDOWS) {
+        winUtilsPath = getQualifiedBinPath("winutils.exe");
+      }
+    } catch (IOException ioe) {
+       LOG.error("Failed to locate the winutils binary in the hadoop binary path",
+         ioe);
+    }
+
+    return winUtilsPath;
+  }
+
+  /** Token separator regex used to parse Shell tool outputs */
+  public static final String TOKEN_SEPARATOR_REGEX
+                = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
+
   private long    interval;   // refresh interval in msec
   private long    lastTime;   // last time the command was performed
   private Map<String, String> environment; // env for the command execution
@@ -144,7 +307,19 @@ abstract public class Shell {
       builder.directory(this.dir);
     }
     
-    process = builder.start();
+    if (Shell.WINDOWS) {
+      synchronized (WindowsProcessLaunchLock) {
+        // To workaround the race condition issue with child processes
+        // inheriting unintended handles during process launch that can
+        // lead to hangs on reading output and error streams, we
+        // serialize process creation. More info available at:
+        // http://support.microsoft.com/kb/315939
+        process = builder.start();
+      }
+    } else {
+      process = builder.start();
+    }
+
     if (timeOutInterval > 0) {
       timeOutTimer = new Timer("Shell command timeout");
       timeoutTimerTask = new ShellTimeoutTimerTask(

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -30,12 +30,16 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.StringTokenizer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.Shell;
 
 import com.google.common.net.InetAddresses;
 
@@ -51,6 +55,27 @@ public class StringUtils {
    */
   public static final int SHUTDOWN_HOOK_PRIORITY = 0;
 
+  /**
+   * Shell environment variables: $ followed by one letter or _ followed by
+   * multiple letters, numbers, or underscores.  The group captures the
+   * environment variable name without the leading $.
+   */
+  public static final Pattern SHELL_ENV_VAR_PATTERN =
+    Pattern.compile("\\$([A-Za-z_]{1}[A-Za-z0-9_]*)");
+
+  /**
+   * Windows environment variables: surrounded by %.  The group captures the
+   * environment variable name without the leading and trailing %.
+   */
+  public static final Pattern WIN_ENV_VAR_PATTERN = Pattern.compile("%(.*?)%");
+
+  /**
+   * Regular expression that matches and captures environment variable names
+   * according to platform-specific rules.
+   */
+  public static final Pattern ENV_VAR_PATTERN = Shell.WINDOWS ?
+    WIN_ENV_VAR_PATTERN : SHELL_ENV_VAR_PATTERN;
+
   /**
    * Make a string representation of the exception.
    * @param e The exception to stringify
@@ -791,6 +816,28 @@ public class StringUtils {
     return sb.toString();
   }
 
+  /**
+   * Concatenates strings, using a separator.
+   *
+   * @param separator to join with
+   * @param strings to join
+   * @return  the joined string
+   */
+  public static String join(CharSequence separator, String[] strings) {
+    // Ideally we don't have to duplicate the code here if array is iterable.
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (String s : strings) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(separator);
+      }
+      sb.append(s);
+    }
+    return sb.toString();
+  }
+
   /**
    * Convert SOME_STUFF to SomeStuff
    *
@@ -806,4 +853,37 @@ public class StringUtils {
 
     return sb.toString();
   }
+
+  /**
+   * Matches a template string against a pattern, replaces matched tokens with
+   * the supplied replacements, and returns the result.  The regular expression
+   * must use a capturing group.  The value of the first capturing group is used
+   * to look up the replacement.  If no replacement is found for the token, then
+   * it is replaced with the empty string.
+   * 
+   * For example, assume template is "%foo%_%bar%_%baz%", pattern is "%(.*?)%",
+   * and replacements contains 2 entries, mapping "foo" to "zoo" and "baz" to
+   * "zaz".  The result returned would be "zoo__zaz".
+   * 
+   * @param template String template to receive replacements
+   * @param pattern Pattern to match for identifying tokens, must use a capturing
+   *   group
+   * @param replacements Map<String, String> mapping tokens identified by the
+   *   capturing group to their replacement values
+   * @return String template with replacements
+   */
+  public static String replaceTokens(String template, Pattern pattern,
+      Map<String, String> replacements) {
+    StringBuffer sb = new StringBuffer();
+    Matcher matcher = pattern.matcher(template);
+    while (matcher.find()) {
+      String replacement = replacements.get(matcher.group(1));
+      if (replacement == null) {
+        replacement = "";
+      }
+      matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+    }
+    matcher.appendTail(sb);
+    return sb.toString();
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java

@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.util;
 
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.Enumeration;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -155,5 +160,7 @@ public class VersionInfo {
     System.out.println("Subversion " + getUrl() + " -r " + getRevision());
     System.out.println("Compiled by " + getUser() + " on " + getDate());
     System.out.println("From source with checksum " + getSrcChecksum());
+    System.out.println("This command was run using " + 
+        ClassUtil.findContainingJar(VersionInfo.class));
   }
 }

hadoop-common-project/hadoop-common/src/main/java/overview.html

@@ -60,9 +60,7 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:</
     Hadoop was been demonstrated on GNU/Linux clusters with 2000 nodes.
   </li>
   <li>
-    Win32 is supported as a <i>development</i> platform. Distributed operation 
-    has not been well tested on Win32, so this is not a <i>production</i> 
-    platform.
+    Windows is also a supported platform.
   </li>  
 </ul>
   
@@ -84,15 +82,6 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:</
   </li>
 </ol>
 
-<h4>Additional requirements for Windows</h4>
-
-<ol>
-  <li>
-    <a href="http://www.cygwin.com/">Cygwin</a> - Required for shell support in 
-    addition to the required software above.
-  </li>
-</ol>
-  
 <h3>Installing Required Software</h3>
 
 <p>If your platform does not have the required software listed above, you
@@ -104,13 +93,6 @@ $ sudo apt-get install ssh<br>
 $ sudo apt-get install rsync<br>
 </pre></blockquote></p>
 
-<p>On Windows, if you did not install the required software when you
-installed cygwin, start the cygwin installer and select the packages:</p>
-<ul>
-  <li>openssh - the "Net" category</li>
-  <li>rsync - the "Net" category</li>
-</ul>
-
 <h2>Getting Started</h2>
 
 <p>First, you need to get a copy of the Hadoop code.</p>

hadoop-common-project/hadoop-common/src/main/native/native.sln

@@ -0,0 +1,48 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "native", "native.vcxproj", "{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Mixed Platforms = Debug|Mixed Platforms
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Mixed Platforms = Release|Mixed Platforms
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.Build.0 = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.ActiveCfg = Release|x64
+		{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

hadoop-common-project/hadoop-common/src/main/native/native.vcxproj

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>native</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <OutDir>..\..\..\target\bin\</OutDir>
+    <IntDir>..\..\..\target\native\$(Configuration)\</IntDir>
+    <TargetName>hadoop</TargetName>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <CompileAs>CompileAsC</CompileAs>
+      <DisableSpecificWarnings>4244</DisableSpecificWarnings>
+    </ClCompile>
+    <Link>
+      <SubSystem>Windows</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalDependencies>Ws2_32.lib;libwinutils.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>..\..\..\target\bin;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Compressor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Decompressor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c" />
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\NativeIO.c" />
+    <ClCompile Include="src\org\apache\hadoop\security\JniBasedUnixGroupsMappingWin.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\bulk_crc32.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCodeLoader.c" />
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" />
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h" />
+    <ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\bulk_crc32.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\crc32c_tables.h" />
+    <ClInclude Include="src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h" />
+    <ClInclude Include="src\org_apache_hadoop.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Resource Files">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\NativeIO.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\bulk_crc32.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\util\NativeCodeLoader.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Compressor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Decompressor.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\org\apache\hadoop\security\JniBasedUnixGroupsMappingWin.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\bulk_crc32.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org\apache\hadoop\util\crc32c_tables.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="src\org_apache_hadoop.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c

@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "org_apache_hadoop_io_compress_bzip2.h"
+#include "org_apache_hadoop_io_compress_bzip2_Bzip2Compressor.h"
+
+static jfieldID Bzip2Compressor_clazz;
+static jfieldID Bzip2Compressor_stream;
+static jfieldID Bzip2Compressor_uncompressedDirectBuf;
+static jfieldID Bzip2Compressor_uncompressedDirectBufOff;
+static jfieldID Bzip2Compressor_uncompressedDirectBufLen;
+static jfieldID Bzip2Compressor_compressedDirectBuf;
+static jfieldID Bzip2Compressor_directBufferSize;
+static jfieldID Bzip2Compressor_finish;
+static jfieldID Bzip2Compressor_finished;
+
+static int (*dlsym_BZ2_bzCompressInit)(bz_stream*, int, int, int);
+static int (*dlsym_BZ2_bzCompress)(bz_stream*, int);
+static int (*dlsym_BZ2_bzCompressEnd)(bz_stream*);
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_initIDs(
+                                 JNIEnv *env, jclass class, jstring libname)
+{
+    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (strcmp(bzlib_name, "system-native") == 0)
+      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    // Load the native library.
+    void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
+    if (!libbz2) {
+        THROW(env, "java/lang/UnsatisfiedLinkError",
+              "Cannot load bzip2 native library");
+        return;
+    }
+
+    // Locate the requisite symbols from libbz2.so.
+    dlerror();                                 // Clear any existing error.
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressInit, env, libbz2,
+                        "BZ2_bzCompressInit");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompress, env, libbz2,
+                        "BZ2_bzCompress");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressEnd, env, libbz2,
+                        "BZ2_bzCompressEnd");
+
+    // Initialize the requisite fieldIds.
+    Bzip2Compressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+                                                     "Ljava/lang/Class;");
+    Bzip2Compressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
+    Bzip2Compressor_finish = (*env)->GetFieldID(env, class, "finish", "Z");
+    Bzip2Compressor_finished = (*env)->GetFieldID(env, class, "finished", "Z");
+    Bzip2Compressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                       "uncompressedDirectBuf",
+                                                       "Ljava/nio/Buffer;");
+    Bzip2Compressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, 
+                                                  "uncompressedDirectBufOff",
+                                                  "I");
+    Bzip2Compressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, 
+                                                  "uncompressedDirectBufLen",
+                                                  "I");
+    Bzip2Compressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                     "compressedDirectBuf", 
+                                                     "Ljava/nio/Buffer;");
+    Bzip2Compressor_directBufferSize = (*env)->GetFieldID(env, class, 
+                                                  "directBufferSize", "I");
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_init(
+            JNIEnv *env, jclass class, jint blockSize, jint workFactor)
+{
+    // Create a bz_stream.
+    bz_stream *stream = malloc(sizeof(bz_stream));
+    if (!stream) {
+        THROW(env, "java/lang/OutOfMemoryError", NULL);
+        return (jlong)0;
+    }
+    memset((void*)stream, 0, sizeof(bz_stream));
+
+    // Initialize stream.
+    int rv = (*dlsym_BZ2_bzCompressInit)(stream, blockSize, 0, workFactor);
+    if (rv != BZ_OK) {
+        // Contingency - Report error by throwing appropriate exceptions.
+        free(stream);
+        stream = NULL;
+        
+        switch (rv) {
+        case BZ_MEM_ERROR: 
+            {
+                THROW(env, "java/lang/OutOfMemoryError", NULL);
+            }
+            break;
+        case BZ_PARAM_ERROR:
+            {
+                THROW(env,
+                      "java/lang/IllegalArgumentException",
+                      NULL);
+            }
+            break;
+        default:
+            {
+                THROW(env, "java/lang/InternalError", NULL);
+            }
+            break;
+        }
+    }
+        
+    return JLONG(stream);
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_deflateBytesDirect(
+        JNIEnv *env, jobject this)
+{
+    // Get members of Bzip2Compressor.
+    bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this, 
+                                              Bzip2Compressor_stream));
+    if (!stream) {
+        THROW(env, "java/lang/NullPointerException", NULL);
+        return (jint)0;
+    } 
+
+    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+                                                 Bzip2Compressor_clazz);
+    jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, 
+                                     Bzip2Compressor_uncompressedDirectBuf);
+    jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, 
+                                   Bzip2Compressor_uncompressedDirectBufOff);
+    jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                   Bzip2Compressor_uncompressedDirectBufLen);
+
+    jobject compressed_direct_buf = (*env)->GetObjectField(env, this, 
+                                   Bzip2Compressor_compressedDirectBuf);
+    jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                 Bzip2Compressor_directBufferSize);
+
+    jboolean finish = (*env)->GetBooleanField(env, this,
+                                              Bzip2Compressor_finish);
+
+    // Get the input and output direct buffers.
+    LOCK_CLASS(env, clazz, "Bzip2Compressor");
+    char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                uncompressed_direct_buf);
+    char* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                compressed_direct_buf);
+    UNLOCK_CLASS(env, clazz, "Bzip2Compressor");
+
+    if (!uncompressed_bytes || !compressed_bytes) {
+        return (jint)0;
+    }
+        
+    // Re-calibrate the bz_stream.
+    stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off;
+    stream->avail_in = uncompressed_direct_buf_len;
+    stream->next_out = compressed_bytes;
+    stream->avail_out = compressed_direct_buf_len;
+        
+    // Compress.
+    int rv = dlsym_BZ2_bzCompress(stream, finish ? BZ_FINISH : BZ_RUN);
+
+    jint no_compressed_bytes = 0;
+    switch (rv) {
+        // Contingency? - Report error by throwing appropriate exceptions.
+    case BZ_STREAM_END:
+        {
+            (*env)->SetBooleanField(env, this,
+                                    Bzip2Compressor_finished,
+                                    JNI_TRUE);
+        } // cascade
+    case BZ_RUN_OK:
+    case BZ_FINISH_OK:
+        {
+            uncompressed_direct_buf_off +=
+                uncompressed_direct_buf_len - stream->avail_in;
+            (*env)->SetIntField(env, this, 
+                                Bzip2Compressor_uncompressedDirectBufOff,
+                                uncompressed_direct_buf_off);
+            (*env)->SetIntField(env, this, 
+                                Bzip2Compressor_uncompressedDirectBufLen,
+                                stream->avail_in);
+            no_compressed_bytes =
+                compressed_direct_buf_len - stream->avail_out;
+        }
+        break;
+    default:
+        {
+            THROW(env, "java/lang/InternalError", NULL);
+        }
+        break;
+    }
+        
+    return no_compressed_bytes;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesRead(
+                            JNIEnv *env, jclass class, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesWritten(
+                                JNIEnv *env, jclass class, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32;
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
+                                JNIEnv *env, jclass class, jlong stream)
+{
+    if (dlsym_BZ2_bzCompressEnd(BZSTREAM(stream)) != BZ_OK) {
+        THROW(env, "java/lang/InternalError", NULL);
+    } else {
+        free(BZSTREAM(stream));
+    }
+}
+
+/**
+ * vim: sw=2: ts=2: et:
+ */
+

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c

@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "org_apache_hadoop_io_compress_bzip2.h"
+#include "org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor.h"
+
+static jfieldID Bzip2Decompressor_clazz;
+static jfieldID Bzip2Decompressor_stream;
+static jfieldID Bzip2Decompressor_compressedDirectBuf;
+static jfieldID Bzip2Decompressor_compressedDirectBufOff;
+static jfieldID Bzip2Decompressor_compressedDirectBufLen;
+static jfieldID Bzip2Decompressor_uncompressedDirectBuf;
+static jfieldID Bzip2Decompressor_directBufferSize;
+static jfieldID Bzip2Decompressor_finished;
+
+static int (*dlsym_BZ2_bzDecompressInit)(bz_stream*, int, int);
+static int (*dlsym_BZ2_bzDecompress)(bz_stream*);
+static int (*dlsym_BZ2_bzDecompressEnd)(bz_stream*);
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_initIDs(
+                                 JNIEnv *env, jclass class, jstring libname)
+{
+    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (strcmp(bzlib_name, "system-native") == 0)
+      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    // Load the native library.
+    void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
+    if (!libbz2) {
+        THROW(env, "java/lang/UnsatisfiedLinkError",
+              "Cannot load bzip2 native library");
+        return;
+    }
+
+    // Locate the requisite symbols from libbz2.so.
+    dlerror();                                 // Clear any existing error.
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressInit, env, libbz2,
+                        "BZ2_bzDecompressInit");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompress, env, libbz2,
+                        "BZ2_bzDecompress");
+    LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressEnd, env, libbz2,
+                        "BZ2_bzDecompressEnd");
+
+    // Initialize the requisite fieldIds.
+    Bzip2Decompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+                                                       "Ljava/lang/Class;");
+    Bzip2Decompressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
+    Bzip2Decompressor_finished = (*env)->GetFieldID(env, class,
+                                                    "finished", "Z");
+    Bzip2Decompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                        "compressedDirectBuf", 
+                                                        "Ljava/nio/Buffer;");
+    Bzip2Decompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, 
+                                                "compressedDirectBufOff", "I");
+    Bzip2Decompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, 
+                                                "compressedDirectBufLen", "I");
+    Bzip2Decompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
+                                                "uncompressedDirectBuf", 
+                                                "Ljava/nio/Buffer;");
+    Bzip2Decompressor_directBufferSize = (*env)->GetFieldID(env, class, 
+                                                "directBufferSize", "I");
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_init(
+                                JNIEnv *env, jclass cls, jint conserveMemory)
+{
+    bz_stream *stream = malloc(sizeof(bz_stream));
+    if (stream == 0) {
+        THROW(env, "java/lang/OutOfMemoryError", NULL);
+        return (jlong)0;
+    } 
+    memset((void*)stream, 0, sizeof(bz_stream));
+    
+    int rv = dlsym_BZ2_bzDecompressInit(stream, 0, conserveMemory);
+
+    if (rv != BZ_OK) {
+        // Contingency - Report error by throwing appropriate exceptions.
+        free(stream);
+        stream = NULL;
+
+        switch (rv) {
+        case BZ_MEM_ERROR:
+            {
+                THROW(env, "java/lang/OutOfMemoryError", NULL);
+            }
+            break;
+        default:
+            {
+                THROW(env, "java/lang/InternalError", NULL);
+            }
+            break;
+        }
+    }
+
+    return JLONG(stream);
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_inflateBytesDirect(
+                        JNIEnv *env, jobject this)
+{
+    // Get members of Bzip2Decompressor.
+    bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this,
+                                                Bzip2Decompressor_stream));
+    if (!stream) {
+        THROW(env, "java/lang/NullPointerException", NULL);
+        return (jint)0;
+    } 
+
+    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+                                                 Bzip2Decompressor_clazz);
+    jarray compressed_direct_buf = (jarray)(*env)->GetObjectField(env,
+                                this, Bzip2Decompressor_compressedDirectBuf);
+    jint compressed_direct_buf_off = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_compressedDirectBufOff);
+    jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_compressedDirectBufLen);
+
+    jarray uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env,
+                                this, Bzip2Decompressor_uncompressedDirectBuf);
+    jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+                                Bzip2Decompressor_directBufferSize);
+
+    // Get the input and output direct buffers.
+    LOCK_CLASS(env, clazz, "Bzip2Decompressor");
+    char* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                compressed_direct_buf);
+    char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+                                                uncompressed_direct_buf);
+    UNLOCK_CLASS(env, clazz, "Bzip2Decompressor");
+
+    if (!compressed_bytes || !uncompressed_bytes) {
+        return (jint)0;
+    }
+        
+    // Re-calibrate the bz_stream.
+    stream->next_in  = compressed_bytes + compressed_direct_buf_off;
+    stream->avail_in  = compressed_direct_buf_len;
+    stream->next_out = uncompressed_bytes;
+    stream->avail_out = uncompressed_direct_buf_len;
+        
+    // Decompress.
+    int rv = dlsym_BZ2_bzDecompress(stream);
+
+    // Contingency? - Report error by throwing appropriate exceptions.
+    int no_decompressed_bytes = 0;      
+    switch (rv) {
+    case BZ_STREAM_END:
+        {
+            (*env)->SetBooleanField(env, this,
+                                    Bzip2Decompressor_finished,
+                                    JNI_TRUE);
+        } // cascade down
+    case BZ_OK:
+        {
+            compressed_direct_buf_off +=
+                compressed_direct_buf_len - stream->avail_in;
+            (*env)->SetIntField(env, this,
+                                Bzip2Decompressor_compressedDirectBufOff, 
+                                compressed_direct_buf_off);
+            (*env)->SetIntField(env, this,
+                                Bzip2Decompressor_compressedDirectBufLen, 
+                                stream->avail_in);
+            no_decompressed_bytes =
+                uncompressed_direct_buf_len - stream->avail_out;
+        }
+        break;
+    case BZ_DATA_ERROR:
+    case BZ_DATA_ERROR_MAGIC:
+        {
+            THROW(env, "java/io/IOException", NULL);
+        }
+        break;
+    case BZ_MEM_ERROR:
+        {
+            THROW(env, "java/lang/OutOfMemoryError", NULL);
+        }
+        break;
+    default:
+        {
+            THROW(env, "java/lang/InternalError", NULL);
+        }
+        break;
+    }
+    
+    return no_decompressed_bytes;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesRead(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32;
+}
+
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesWritten(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    const bz_stream* strm = BZSTREAM(stream);
+    return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32;
+}
+
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getRemaining(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    return (BZSTREAM(stream))->avail_in;
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_end(
+                                JNIEnv *env, jclass cls, jlong stream)
+{
+    if (dlsym_BZ2_bzDecompressEnd(BZSTREAM(stream)) != BZ_OK) {
+        THROW(env, "java/lang/InternalError", 0);
+    } else {
+        free(BZSTREAM(stream));
+    }
+}
+
+/**
+ * vim: sw=2: ts=2: et:
+ */
+

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H
+#define ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H
+
+#include <config.h>
+#include <stddef.h>
+#include <bzlib.h>
+#include <dlfcn.h>
+#include <jni.h>
+
+#include "org_apache_hadoop.h"
+
+#define HADOOP_BZIP2_LIBRARY "libbz2.so.1"
+
+
+/* A helper macro to convert the java 'stream-handle' to a bz_stream pointer. */
+#define BZSTREAM(stream) ((bz_stream*)((ptrdiff_t)(stream)))
+
+/* A helper macro to convert the bz_stream pointer to the java 'stream-handle'. */
+#define JLONG(stream) ((jlong)((ptrdiff_t)(stream)))
+
+#endif //ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c

@@ -16,10 +16,14 @@
  * limitations under the License.
  */
 
-#include "config.h"
+
 #include "org_apache_hadoop.h"
 #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h"
 
+#ifdef UNIX
+#include "config.h"
+#endif // UNIX
+
 //****************************
 // Simple Functions
 //****************************
@@ -61,6 +65,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_init
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char* uncompressed_bytes;
+  char *compressed_bytes;
+
   // Get members of Lz4Compressor
   jobject clazz = (*env)->GetStaticObjectField(env, thisj, Lz4Compressor_clazz);
   jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf);
@@ -70,7 +77,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "Lz4Compressor");
-  const char* uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Compressor");
 
   if (uncompressed_bytes == 0) {
@@ -79,7 +86,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "Lz4Compressor");
-  char* compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Compressor");
 
   if (compressed_bytes == 0) {

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c

@@ -16,10 +16,13 @@
  * limitations under the License.
  */
 
-#include "config.h"
 #include "org_apache_hadoop.h"
 #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h"
 
+#ifdef UNIX
+#include "config.h"
+#endif // UNIX
+
 int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int isize, int maxOutputSize);
 
 /*
@@ -58,6 +61,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_in
 
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_decompressBytesDirect
 (JNIEnv *env, jobject thisj){
+  const char *compressed_bytes;
+  char *uncompressed_bytes;
+
   // Get members of Lz4Decompressor
   jobject clazz = (*env)->GetStaticObjectField(env,thisj, Lz4Decompressor_clazz);
   jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_compressedDirectBuf);
@@ -67,7 +73,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de
 
   // Get the input direct buffer
   LOCK_CLASS(env, clazz, "Lz4Decompressor");
-  const char* compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+  compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
 
   if (compressed_bytes == 0) {
@@ -76,7 +82,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de
 
   // Get the output direct buffer
   LOCK_CLASS(env, clazz, "Lz4Decompressor");
-  char* uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+  uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
   UNLOCK_CLASS(env, clazz, "Lz4Decompressor");
 
   if (uncompressed_bytes == 0) {

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c

@@ -16,12 +16,18 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
+#endif // UNIX
+
 #include "org_apache_hadoop_io_compress_snappy.h"
 #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
 
@@ -81,7 +87,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (uncompressed_bytes == 0) {
-    return 0;
+    return (jint)0;
   }
 
   // Get the output direct buffer
@@ -90,7 +96,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   UNLOCK_CLASS(env, clazz, "SnappyCompressor");
 
   if (compressed_bytes == 0) {
-    return 0;
+    return (jint)0;
   }
 
   /* size_t should always be 4 bytes or larger. */
@@ -109,3 +115,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   (*env)->SetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen, 0);
   return (jint)buf_len;
 }
+
+#endif //define HADOOP_SNAPPY_LIBRARY

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c

@@ -16,12 +16,18 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
 #include "config.h"
+#include <dlfcn.h>
+#endif
+
 #include "org_apache_hadoop_io_compress_snappy.h"
 #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
 
@@ -103,3 +109,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres
 
   return (jint)uncompressed_direct_buf_len;
 }
+
+#endif //define HADOOP_SNAPPY_LIBRARY

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c

@@ -16,12 +16,15 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
+#endif
+
 #include "org_apache_hadoop_io_compress_zlib.h"
 #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h"
 
@@ -35,48 +38,124 @@ static jfieldID ZlibCompressor_directBufferSize;
 static jfieldID ZlibCompressor_finish;
 static jfieldID ZlibCompressor_finished;
 
+#ifdef UNIX
 static int (*dlsym_deflateInit2_)(z_streamp, int, int, int, int, int, const char *, int);
 static int (*dlsym_deflate)(z_streamp, int);
 static int (*dlsym_deflateSetDictionary)(z_streamp, const Bytef *, uInt);
 static int (*dlsym_deflateReset)(z_streamp);
 static int (*dlsym_deflateEnd)(z_streamp);
+#endif
+
+#ifdef WINDOWS
+#include <Strsafe.h>
+typedef int (__cdecl *__dlsym_deflateInit2_) (z_streamp, int, int, int, int, int, const char *, int);
+typedef int (__cdecl *__dlsym_deflate) (z_streamp, int);
+typedef int (__cdecl *__dlsym_deflateSetDictionary) (z_streamp, const Bytef *, uInt);
+typedef int (__cdecl *__dlsym_deflateReset) (z_streamp);
+typedef int (__cdecl *__dlsym_deflateEnd) (z_streamp);
+static __dlsym_deflateInit2_ dlsym_deflateInit2_;
+static __dlsym_deflate dlsym_deflate;
+static __dlsym_deflateSetDictionary dlsym_deflateSetDictionary;
+static __dlsym_deflateReset dlsym_deflateReset;
+static __dlsym_deflateEnd dlsym_deflateEnd;
+
+// Try to load zlib.dll from the dir where hadoop.dll is located.
+HANDLE LoadZlibTryHadoopNativeDir() {
+  HMODULE libz = NULL;
+  PCWSTR HADOOP_DLL = L"hadoop.dll";
+  size_t HADOOP_DLL_LEN = 10;
+  WCHAR path[MAX_PATH] = { 0 };
+  BOOL isPathValid = FALSE;
+
+  // Get hadoop.dll full path
+  HMODULE hModule = GetModuleHandle(HADOOP_DLL);
+  if (hModule != NULL) {
+    if (GetModuleFileName(hModule, path, MAX_PATH) > 0) {
+      size_t size = 0;
+      if (StringCchLength(path, MAX_PATH, &size) == S_OK) {
+
+        // Update path variable to have the full path to the zlib.dll
+        size = size - HADOOP_DLL_LEN;
+        if (size >= 0) {
+          path[size] = L'\0';
+          if (StringCchCat(path, MAX_PATH, HADOOP_ZLIB_LIBRARY) == S_OK) {
+            isPathValid = TRUE;
+          }
+        }
+      }
+    }
+  }
+
+  if (isPathValid) {
+    libz = LoadLibrary(path);
+  }
+
+  // fallback to system paths
+  if (!libz) {
+    libz = LoadLibrary(HADOOP_ZLIB_LIBRARY);
+  }
+
+  return libz;
+}
+#endif
 
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_initIDs(
 	JNIEnv *env, jclass class
 	) {
+#ifdef UNIX
 	// Load libz.so
 	void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
-	if (!libz) {
+  if (!libz) {
 		THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load libz.so");
 	  	return;
 	}
+#endif
+
+#ifdef WINDOWS
+  HMODULE libz = LoadZlibTryHadoopNativeDir();
 
+  if (!libz) {
+		THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zlib1.dll");
+    return;
+	}
+#endif
+
+#ifdef UNIX
 	// Locate the requisite symbols from libz.so
 	dlerror();                                 // Clear any existing error
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset");
-	LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset");
+  LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd");
+#endif
+
+#ifdef WINDOWS
+  LOAD_DYNAMIC_SYMBOL(__dlsym_deflateInit2_, dlsym_deflateInit2_, env, libz, "deflateInit2_");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflate, dlsym_deflate, env, libz, "deflate");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateSetDictionary, dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateReset, dlsym_deflateReset, env, libz, "deflateReset");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_deflateEnd, dlsym_deflateEnd, env, libz, "deflateEnd");
+#endif
 
 	// Initialize the requisite fieldIds
-    ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+    ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz",
                                                       "Ljava/lang/Class;");
     ZlibCompressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
     ZlibCompressor_finish = (*env)->GetFieldID(env, class, "finish", "Z");
     ZlibCompressor_finished = (*env)->GetFieldID(env, class, "finished", "Z");
-    ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
-    									"uncompressedDirectBuf", 
+    ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class,
+        "uncompressedDirectBuf",
     									"Ljava/nio/Buffer;");
-    ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class,
     										"uncompressedDirectBufOff", "I");
-    ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class,
     										"uncompressedDirectBufLen", "I");
-    ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
-    									"compressedDirectBuf", 
+    ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class,
+                      "compressedDirectBuf",
     									"Ljava/nio/Buffer;");
-    ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class, 
+    ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class,
     										"directBufferSize", "I");
 }
 
@@ -84,7 +163,9 @@ JNIEXPORT jlong JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
 	JNIEnv *env, jclass class, jint level, jint strategy, jint windowBits
 	) {
-	// Create a z_stream
+    int rv = 0;
+    static const int memLevel = 8; 							// See zconf.h
+	  // Create a z_stream
     z_stream *stream = malloc(sizeof(z_stream));
     if (!stream) {
 		THROW(env, "java/lang/OutOfMemoryError", NULL);
@@ -93,17 +174,16 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
     memset((void*)stream, 0, sizeof(z_stream));
 
 	// Initialize stream
-	static const int memLevel = 8; 							// See zconf.h
-    int rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits,
+    rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits,
     			memLevel, strategy, ZLIB_VERSION, sizeof(z_stream));
-    			
+
     if (rv != Z_OK) {
 	    // Contingency - Report error by throwing appropriate exceptions
 	    free(stream);
 	    stream = NULL;
-	
+
 		switch (rv) {
-			case Z_MEM_ERROR: 
+			case Z_MEM_ERROR:
 			    {
 		    		THROW(env, "java/lang/OutOfMemoryError", NULL);
 			    }
@@ -120,27 +200,28 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init(
 		    break;
 	    }
 	}
-	
+
     return JLONG(stream);
 }
 
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_setDictionary(
-	JNIEnv *env, jclass class, jlong stream, 
+	JNIEnv *env, jclass class, jlong stream,
 	jarray b, jint off, jint len
 	) {
+    int rv = 0;
     Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0);
     if (!buf) {
         return;
     }
-    int rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len);
+    rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len);
     (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0);
-    
+
     if (rv != Z_OK) {
     	// Contingency - Report error by throwing appropriate exceptions
 	    switch (rv) {
 		    case Z_STREAM_ERROR:
-			{	
+			{
 		    	THROW(env, "java/lang/IllegalArgumentException", NULL);
 			}
 			break;
@@ -157,75 +238,85 @@ JNIEXPORT jint JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect(
 	JNIEnv *env, jobject this
 	) {
+    jobject clazz = NULL;
+    jobject uncompressed_direct_buf = NULL;
+    jint uncompressed_direct_buf_off = 0;
+    jint uncompressed_direct_buf_len = 0;
+    jobject compressed_direct_buf = NULL;
+    jint compressed_direct_buf_len = 0;
+    jboolean finish;
+    Bytef* uncompressed_bytes = NULL;
+    Bytef* compressed_bytes = NULL;
+    int rv = 0;
+    jint no_compressed_bytes = 0;
 	// Get members of ZlibCompressor
     z_stream *stream = ZSTREAM(
-    						(*env)->GetLongField(env, this, 
+                (*env)->GetLongField(env, this,
     									ZlibCompressor_stream)
     					);
     if (!stream) {
 		THROW(env, "java/lang/NullPointerException", NULL);
 		return (jint)0;
-    } 
+    }
 
     // Get members of ZlibCompressor
-    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+    clazz = (*env)->GetStaticObjectField(env, this,
                                                  ZlibCompressor_clazz);
-	jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, 
+	uncompressed_direct_buf = (*env)->GetObjectField(env, this,
 									ZlibCompressor_uncompressedDirectBuf);
-	jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, 
+	uncompressed_direct_buf_off = (*env)->GetIntField(env, this,
 									ZlibCompressor_uncompressedDirectBufOff);
-	jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	uncompressed_direct_buf_len = (*env)->GetIntField(env, this,
 									ZlibCompressor_uncompressedDirectBufLen);
 
-	jobject compressed_direct_buf = (*env)->GetObjectField(env, this, 
+	compressed_direct_buf = (*env)->GetObjectField(env, this,
 									ZlibCompressor_compressedDirectBuf);
-	jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	compressed_direct_buf_len = (*env)->GetIntField(env, this,
 									ZlibCompressor_directBufferSize);
 
-	jboolean finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish);
+	finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish);
 
     // Get the input direct buffer
     LOCK_CLASS(env, clazz, "ZlibCompressor");
-	Bytef* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+    uncompressed_bytes = (*env)->GetDirectBufferAddress(env,
 											uncompressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibCompressor");
-    
+
   	if (uncompressed_bytes == 0) {
     	return (jint)0;
 	}
-	
+
     // Get the output direct buffer
     LOCK_CLASS(env, clazz, "ZlibCompressor");
-	Bytef* compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+    compressed_bytes = (*env)->GetDirectBufferAddress(env,
 										compressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibCompressor");
 
   	if (compressed_bytes == 0) {
 		return (jint)0;
 	}
-	
+
 	// Re-calibrate the z_stream
   	stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off;
   	stream->next_out = compressed_bytes;
   	stream->avail_in = uncompressed_direct_buf_len;
-	stream->avail_out = compressed_direct_buf_len;
-	
+    stream->avail_out = compressed_direct_buf_len;
+
 	// Compress
-	int rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
+	rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
 
-	jint no_compressed_bytes = 0;
 	switch (rv) {
     	// Contingency? - Report error by throwing appropriate exceptions
   		case Z_STREAM_END:
   		{
   			(*env)->SetBooleanField(env, this, ZlibCompressor_finished, JNI_TRUE);
   		} // cascade
-	  	case Z_OK: 
+      case Z_OK:
 	  	{
 	  		uncompressed_direct_buf_off += uncompressed_direct_buf_len - stream->avail_in;
-			(*env)->SetIntField(env, this, 
+			(*env)->SetIntField(env, this,
 						ZlibCompressor_uncompressedDirectBufOff, uncompressed_direct_buf_off);
-			(*env)->SetIntField(env, this, 
+			(*env)->SetIntField(env, this,
 						ZlibCompressor_uncompressedDirectBufLen, stream->avail_in);
 			no_compressed_bytes = compressed_direct_buf_len - stream->avail_out;
 	  	}
@@ -238,7 +329,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect(
 		}
 		break;
   	}
-  	
+
   	return no_compressed_bytes;
 }
 

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c

@@ -16,12 +16,15 @@
  * limitations under the License.
  */
 
-#include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
+#endif
+
 #include "org_apache_hadoop_io_compress_zlib.h"
 #include "org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h"
 
@@ -35,48 +38,88 @@ static jfieldID ZlibDecompressor_directBufferSize;
 static jfieldID ZlibDecompressor_needDict;
 static jfieldID ZlibDecompressor_finished;
 
+#ifdef UNIX
 static int (*dlsym_inflateInit2_)(z_streamp, int, const char *, int);
 static int (*dlsym_inflate)(z_streamp, int);
 static int (*dlsym_inflateSetDictionary)(z_streamp, const Bytef *, uInt);
 static int (*dlsym_inflateReset)(z_streamp);
 static int (*dlsym_inflateEnd)(z_streamp);
+#endif
+
+#ifdef WINDOWS
+#include <Strsafe.h>
+typedef int (__cdecl *__dlsym_inflateInit2_)(z_streamp, int, const char *, int);
+typedef int (__cdecl *__dlsym_inflate)(z_streamp, int);
+typedef int (__cdecl *__dlsym_inflateSetDictionary)(z_streamp, const Bytef *, uInt);
+typedef int (__cdecl *__dlsym_inflateReset)(z_streamp);
+typedef int (__cdecl *__dlsym_inflateEnd)(z_streamp);
+static __dlsym_inflateInit2_ dlsym_inflateInit2_;
+static __dlsym_inflate dlsym_inflate;
+static __dlsym_inflateSetDictionary dlsym_inflateSetDictionary;
+static __dlsym_inflateReset dlsym_inflateReset;
+static __dlsym_inflateEnd dlsym_inflateEnd;
+extern HANDLE LoadZlibTryHadoopNativeDir();
+#endif
 
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_initIDs(
-	JNIEnv *env, jclass class
+JNIEnv *env, jclass class
 	) {
 	// Load libz.so
-    void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+#ifdef UNIX
+  void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
 	if (!libz) {
 	  THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load libz.so");
 	  return;
-	} 
+	}
+#endif
+
+#ifdef WINDOWS
+  HMODULE libz = LoadZlibTryHadoopNativeDir();
+
+	if (!libz) {
+	  THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zlib1.dll");
+	  return;
+	}
+#endif
+
 
 	// Locate the requisite symbols from libz.so
+#ifdef UNIX
 	dlerror();                                 // Clear any existing error
 	LOAD_DYNAMIC_SYMBOL(dlsym_inflateInit2_, env, libz, "inflateInit2_");
 	LOAD_DYNAMIC_SYMBOL(dlsym_inflate, env, libz, "inflate");
 	LOAD_DYNAMIC_SYMBOL(dlsym_inflateSetDictionary, env, libz, "inflateSetDictionary");
 	LOAD_DYNAMIC_SYMBOL(dlsym_inflateReset, env, libz, "inflateReset");
 	LOAD_DYNAMIC_SYMBOL(dlsym_inflateEnd, env, libz, "inflateEnd");
+#endif
+
+#ifdef WINDOWS
+	LOAD_DYNAMIC_SYMBOL(__dlsym_inflateInit2_, dlsym_inflateInit2_, env, libz, "inflateInit2_");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_inflate, dlsym_inflate, env, libz, "inflate");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_inflateSetDictionary, dlsym_inflateSetDictionary, env, libz, "inflateSetDictionary");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_inflateReset, dlsym_inflateReset, env, libz, "inflateReset");
+	LOAD_DYNAMIC_SYMBOL(__dlsym_inflateEnd, dlsym_inflateEnd, env, libz, "inflateEnd");
+#endif
+
 
-	// Initialize the requisite fieldIds
-    ZlibDecompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", 
+  // Initialize the requisite fieldIds
+    ZlibDecompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz",
                                                       "Ljava/lang/Class;");
     ZlibDecompressor_stream = (*env)->GetFieldID(env, class, "stream", "J");
     ZlibDecompressor_needDict = (*env)->GetFieldID(env, class, "needDict", "Z");
     ZlibDecompressor_finished = (*env)->GetFieldID(env, class, "finished", "Z");
-    ZlibDecompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, 
-    											"compressedDirectBuf", 
+    ZlibDecompressor_compressedDirectBuf = (*env)->GetFieldID(env, class,
+                          "compressedDirectBuf",
     											"Ljava/nio/Buffer;");
-    ZlibDecompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, 
+    ZlibDecompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class,
     										"compressedDirectBufOff", "I");
-    ZlibDecompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, 
+    ZlibDecompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class,
     										"compressedDirectBufLen", "I");
-    ZlibDecompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, 
-    											"uncompressedDirectBuf", 
+    ZlibDecompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class,
+                          "uncompressedDirectBuf",
     											"Ljava/nio/Buffer;");
-    ZlibDecompressor_directBufferSize = (*env)->GetFieldID(env, class, 
+    ZlibDecompressor_directBufferSize = (*env)->GetFieldID(env, class,
     											"directBufferSize", "I");
 }
 
@@ -84,21 +127,22 @@ JNIEXPORT jlong JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_init(
 	JNIEnv *env, jclass cls, jint windowBits
 	) {
+    int rv = 0;
     z_stream *stream = malloc(sizeof(z_stream));
     memset((void*)stream, 0, sizeof(z_stream));
 
     if (stream == 0) {
 		THROW(env, "java/lang/OutOfMemoryError", NULL);
 		return (jlong)0;
-    } 
-    
-    int rv = dlsym_inflateInit2_(stream, windowBits, ZLIB_VERSION, sizeof(z_stream));
+    }
+
+    rv = dlsym_inflateInit2_(stream, windowBits, ZLIB_VERSION, sizeof(z_stream));
 
 	if (rv != Z_OK) {
 	    // Contingency - Report error by throwing appropriate exceptions
 		free(stream);
 		stream = NULL;
-		
+
 		switch (rv) {
 		 	case Z_MEM_ERROR:
 		 	{
@@ -112,7 +156,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_init(
 	  		break;
 		}
 	}
-	
+
 	return JLONG(stream);
 }
 
@@ -121,21 +165,22 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_setDictionary(
 	JNIEnv *env, jclass cls, jlong stream,
 	jarray b, jint off, jint len
 	) {
+    int rv = 0;
     Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0);
     if (!buf) {
 		THROW(env, "java/lang/InternalError", NULL);
         return;
     }
-    int rv = dlsym_inflateSetDictionary(ZSTREAM(stream), buf + off, len);
+    rv = dlsym_inflateSetDictionary(ZSTREAM(stream), buf + off, len);
     (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0);
-    
+
     if (rv != Z_OK) {
 	    // Contingency - Report error by throwing appropriate exceptions
 		switch (rv) {
 		    case Z_STREAM_ERROR:
 	    	case Z_DATA_ERROR:
 			{
-				THROW(env, "java/lang/IllegalArgumentException", 
+				THROW(env, "java/lang/IllegalArgumentException",
 					(ZSTREAM(stream))->msg);
 			}
 			break;
@@ -152,62 +197,71 @@ JNIEXPORT jint JNICALL
 Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect(
 	JNIEnv *env, jobject this
 	) {
+    jobject clazz = NULL;
+    jarray compressed_direct_buf = NULL;
+    jint compressed_direct_buf_off = 0;
+    jint compressed_direct_buf_len = 0;
+    jarray uncompressed_direct_buf = NULL;
+    jint uncompressed_direct_buf_len = 0;
+    Bytef *compressed_bytes = NULL;
+    Bytef *uncompressed_bytes = NULL;
+    int rv = 0;
+    int no_decompressed_bytes = 0;
 	// Get members of ZlibDecompressor
     z_stream *stream = ZSTREAM(
-    						(*env)->GetLongField(env, this, 
+                (*env)->GetLongField(env, this,
     									ZlibDecompressor_stream)
     					);
     if (!stream) {
 		THROW(env, "java/lang/NullPointerException", NULL);
 		return (jint)0;
-    } 
+    }
 
     // Get members of ZlibDecompressor
-    jobject clazz = (*env)->GetStaticObjectField(env, this, 
+    clazz = (*env)->GetStaticObjectField(env, this,
                                                  ZlibDecompressor_clazz);
-	jarray compressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, 
+	compressed_direct_buf = (jarray)(*env)->GetObjectField(env, this,
 											ZlibDecompressor_compressedDirectBuf);
-	jint compressed_direct_buf_off = (*env)->GetIntField(env, this, 
+	compressed_direct_buf_off = (*env)->GetIntField(env, this,
 									ZlibDecompressor_compressedDirectBufOff);
-	jint compressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	compressed_direct_buf_len = (*env)->GetIntField(env, this,
 									ZlibDecompressor_compressedDirectBufLen);
 
-	jarray uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, 
+	uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env, this,
 											ZlibDecompressor_uncompressedDirectBuf);
-	jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, 
+	uncompressed_direct_buf_len = (*env)->GetIntField(env, this,
 										ZlibDecompressor_directBufferSize);
 
     // Get the input direct buffer
     LOCK_CLASS(env, clazz, "ZlibDecompressor");
-	Bytef *compressed_bytes = (*env)->GetDirectBufferAddress(env, 
+	compressed_bytes = (*env)->GetDirectBufferAddress(env,
 										compressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibDecompressor");
-    
+
 	if (!compressed_bytes) {
 	    return (jint)0;
 	}
-	
+
     // Get the output direct buffer
     LOCK_CLASS(env, clazz, "ZlibDecompressor");
-	Bytef *uncompressed_bytes = (*env)->GetDirectBufferAddress(env, 
+	uncompressed_bytes = (*env)->GetDirectBufferAddress(env,
 											uncompressed_direct_buf);
     UNLOCK_CLASS(env, clazz, "ZlibDecompressor");
 
 	if (!uncompressed_bytes) {
 	    return (jint)0;
 	}
-	
+
 	// Re-calibrate the z_stream
 	stream->next_in  = compressed_bytes + compressed_direct_buf_off;
 	stream->next_out = uncompressed_bytes;
 	stream->avail_in  = compressed_direct_buf_len;
 	stream->avail_out = uncompressed_direct_buf_len;
-	
+
 	// Decompress
-	int rv = dlsym_inflate(stream, Z_PARTIAL_FLUSH);
+	rv = dlsym_inflate(stream, Z_PARTIAL_FLUSH);
 
 	// Contingency? - Report error by throwing appropriate exceptions
-	int no_decompressed_bytes = 0;	
 	switch (rv) {
 		case Z_STREAM_END:
 		{
@@ -216,9 +270,9 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect(
 		case Z_OK:
 		{
 		    compressed_direct_buf_off += compressed_direct_buf_len - stream->avail_in;
-		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, 
+		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff,
 		    			compressed_direct_buf_off);
-		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, 
+		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen,
 		    			stream->avail_in);
 		    no_decompressed_bytes = uncompressed_direct_buf_len - stream->avail_out;
 		}
@@ -227,9 +281,9 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect(
 		{
 		    (*env)->SetBooleanField(env, this, ZlibDecompressor_needDict, JNI_TRUE);
 		    compressed_direct_buf_off += compressed_direct_buf_len - stream->avail_in;
-		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, 
+		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff,
 		    			compressed_direct_buf_off);
-		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, 
+		    (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen,
 		    			stream->avail_in);
 		}
 		break;
@@ -251,7 +305,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect(
 		}
 		break;
     }
-    
+
     return no_decompressed_bytes;
 }
 
@@ -299,4 +353,3 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_end(
 /**
  * vim: sw=2: ts=2: et:
  */
-

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h

@@ -19,14 +19,23 @@
 #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H
 #define ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H
 
-#include <dlfcn.h>
-#include <jni.h>
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
+#include <config.h>
 #include <stddef.h>
-#include <zconf.h>
 #include <zlib.h>
+#include <zconf.h>
+#include <dlfcn.h>
+#include <jni.h>
+#endif
 
-#include "config.h"
-#include "org_apache_hadoop.h"
+#ifdef WINDOWS
+#include <jni.h>
+#define HADOOP_ZLIB_LIBRARY L"zlib1.dll"
+#include <zlib.h>
+#include <zconf.h>
+#endif
 
 /* A helper macro to convert the java 'stream-handle' to a z_stream pointer. */
 #define ZSTREAM(stream) ((z_stream*)((ptrdiff_t)(stream)))

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c

@@ -18,6 +18,10 @@
 
 #define _GNU_SOURCE
 
+#include "org_apache_hadoop.h"
+#include "org_apache_hadoop_io_nativeio_NativeIO.h"
+
+#ifdef UNIX
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -31,14 +35,19 @@
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <unistd.h>
-
 #include "config.h"
-#include "org_apache_hadoop.h"
-#include "org_apache_hadoop_io_nativeio_NativeIO.h"
+#endif
+
+#ifdef WINDOWS
+#include <assert.h>
+#include <Windows.h>
+#include "winutils.h"
+#endif
+
 #include "file_descriptor.h"
 #include "errno_enum.h"
 
-// the NativeIO$Stat inner class and its constructor
+// the NativeIO$POSIX$Stat inner class and its constructor
 static jclass stat_clazz;
 static jmethodID stat_ctor;
 
@@ -53,26 +62,32 @@ static jobject pw_lock_object;
 
 // Internal functions
 static void throw_ioe(JNIEnv* env, int errnum);
+#ifdef UNIX
 static ssize_t get_pw_buflen();
+#endif
 
 /**
  * Returns non-zero if the user has specified that the system
  * has non-threadsafe implementations of getpwuid_r or getgrgid_r.
  **/
 static int workaround_non_threadsafe_calls(JNIEnv *env, jclass clazz) {
-  jfieldID needs_workaround_field = (*env)->GetStaticFieldID(env, clazz,
-    "workaroundNonThreadSafePasswdCalls", "Z");
+  jboolean result;
+  jfieldID needs_workaround_field = (*env)->GetStaticFieldID(
+    env, clazz,
+    "workaroundNonThreadSafePasswdCalls",
+    "Z");
   PASS_EXCEPTIONS_RET(env, 0);
   assert(needs_workaround_field);
 
-  jboolean result = (*env)->GetStaticBooleanField(
+  result = (*env)->GetStaticBooleanField(
     env, clazz, needs_workaround_field);
   return result;
 }
 
+#ifdef UNIX
 static void stat_init(JNIEnv *env, jclass nativeio_class) {
   // Init Stat
-  jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$Stat");
+  jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat");
   if (!clazz) {
     return; // exception has been raised
   }
@@ -85,6 +100,7 @@ static void stat_init(JNIEnv *env, jclass nativeio_class) {
   if (!stat_ctor) {
     return; // exception has been raised
   }
+
   jclass obj_class = (*env)->FindClass(env, "java/lang/Object");
   if (!obj_class) {
     return; // exception has been raised
@@ -99,6 +115,7 @@ static void stat_init(JNIEnv *env, jclass nativeio_class) {
     pw_lock_object = (*env)->NewObject(env, obj_class, obj_ctor);
     PASS_EXCEPTIONS(env);
     pw_lock_object = (*env)->NewGlobalRef(env, pw_lock_object);
+
     PASS_EXCEPTIONS(env);
   }
 }
@@ -113,6 +130,7 @@ static void stat_deinit(JNIEnv *env) {
     pw_lock_object = NULL;
   }
 }
+#endif
 
 static void nioe_init(JNIEnv *env) {
   // Init NativeIOException
@@ -121,8 +139,15 @@ static void nioe_init(JNIEnv *env) {
   PASS_EXCEPTIONS(env);
 
   nioe_clazz = (*env)->NewGlobalRef(env, nioe_clazz);
+#ifdef UNIX
   nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "<init>",
     "(Ljava/lang/String;Lorg/apache/hadoop/io/nativeio/Errno;)V");
+#endif
+
+#ifdef WINDOWS
+  nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "<init>",
+    "(Ljava/lang/String;I)V");
+#endif
 }
 
 static void nioe_deinit(JNIEnv *env) {
@@ -143,32 +168,46 @@ static void nioe_deinit(JNIEnv *env) {
 JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
 	JNIEnv *env, jclass clazz) {
-
+#ifdef UNIX
   stat_init(env, clazz);
   PASS_EXCEPTIONS_GOTO(env, error);
+#endif
   nioe_init(env);
   PASS_EXCEPTIONS_GOTO(env, error);
   fd_init(env);
   PASS_EXCEPTIONS_GOTO(env, error);
+#ifdef UNIX
   errno_enum_init(env);
   PASS_EXCEPTIONS_GOTO(env, error);
+#endif
   return;
 error:
   // these are all idempodent and safe to call even if the
   // class wasn't initted yet
+#ifdef UNIX
   stat_deinit(env);
+#endif
   nioe_deinit(env);
   fd_deinit(env);
+#ifdef UNIX
   errno_enum_deinit(env);
+#endif
 }
 
 /*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_POSIX
+ * Method:    fstat
+ * Signature: (Ljava/io/FileDescriptor;)Lorg/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat;
  * public static native Stat fstat(FileDescriptor fd);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
 JNIEXPORT jobject JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_fstat(
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_fstat(
   JNIEnv *env, jclass clazz, jobject fd_object)
 {
+#ifdef UNIX
   jobject ret = NULL;
 
   int fd = fd_get(env, fd_object);
@@ -187,14 +226,26 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_fstat(
 
 cleanup:
   return ret;
+#endif
+
+#ifdef WINDOWS
+  THROW(env, "java/io/IOException",
+    "The function POSIX.fstat() is not supported on Windows");
+  return NULL;
+#endif
 }
 
+
+
 /**
  * public static native void posix_fadvise(
  *   FileDescriptor fd, long offset, long len, int flags);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
 JNIEXPORT void JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_posix_1fadvise(
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_posix_1fadvise(
   JNIEnv *env, jclass clazz,
   jobject fd_object, jlong offset, jlong len, jint flags)
 {
@@ -240,9 +291,12 @@ static int manual_sync_file_range (int fd, __off64_t from, __off64_t to, unsigne
 /**
  * public static native void sync_file_range(
  *   FileDescriptor fd, long offset, long len, int flags);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
 JNIEXPORT void JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_sync_1file_1range(
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_sync_1file_1range(
   JNIEnv *env, jclass clazz,
   jobject fd_object, jlong offset, jlong len, jint flags)
 {
@@ -284,13 +338,20 @@ static int toFreeBSDFlags(int flags)
 #endif
 
 /*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_POSIX
+ * Method:    open
+ * Signature: (Ljava/lang/String;II)Ljava/io/FileDescriptor;
  * public static native FileDescriptor open(String path, int flags, int mode);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
 JNIEXPORT jobject JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_open(
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_open(
   JNIEnv *env, jclass clazz, jstring j_path,
   jint flags, jint mode)
 {
+#ifdef UNIX
 #ifdef __FreeBSD__
   flags = toFreeBSDFlags(flags);
 #endif
@@ -318,16 +379,90 @@ cleanup:
     (*env)->ReleaseStringUTFChars(env, j_path, path);
   }
   return ret;
+#endif
+
+#ifdef WINDOWS
+  THROW(env, "java/io/IOException",
+    "The function POSIX.open() is not supported on Windows");
+  return NULL;
+#endif
 }
 
-/**
- * public static native void chmod(String path, int mode) throws IOException;
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    createFile
+ * Signature: (Ljava/lang/String;JJJ)Ljava/io/FileDescriptor;
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
-JNIEXPORT void JNICALL
-Java_org_apache_hadoop_io_nativeio_NativeIO_chmod(
-  JNIEnv *env, jclass clazz, jstring j_path,
-  jint mode)
+JNIEXPORT jobject JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_createFile
+  (JNIEnv *env, jclass clazz, jstring j_path,
+  jlong desiredAccess, jlong shareMode, jlong creationDisposition)
+{
+#ifdef UNIX
+  THROW(env, "java/io/IOException",
+    "The function Windows.createFile() is not supported on Unix");
+  return NULL;
+#endif
+
+#ifdef WINDOWS
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  BOOL isSymlink = FALSE;
+  BOOL isJunction = FALSE;
+  DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS;
+  jobject ret = (jobject) NULL;
+  HANDLE hFile = INVALID_HANDLE_VALUE;
+  WCHAR *path = (WCHAR *) (*env)->GetStringChars(env, j_path, (jboolean*)NULL);
+  if (path == NULL) goto cleanup;
+
+  // Set the flag for a symbolic link or a junctions point only when it exists.
+  // According to MSDN if the call to CreateFile() function creates a file,
+  // there is no change in behavior. So we do not throw if no file is found.
+  //
+  dwRtnCode = SymbolicLinkCheck(path, &isSymlink);
+  if (dwRtnCode != ERROR_SUCCESS && dwRtnCode != ERROR_FILE_NOT_FOUND) {
+    throw_ioe(env, dwRtnCode);
+    goto cleanup;
+  }
+  dwRtnCode = JunctionPointCheck(path, &isJunction);
+  if (dwRtnCode != ERROR_SUCCESS && dwRtnCode != ERROR_FILE_NOT_FOUND) {
+    throw_ioe(env, dwRtnCode);
+    goto cleanup;
+  }
+  if (isSymlink || isJunction)
+    dwFlagsAndAttributes |= FILE_FLAG_OPEN_REPARSE_POINT;
+
+  hFile = CreateFile(path,
+    (DWORD) desiredAccess,
+    (DWORD) shareMode,
+    (LPSECURITY_ATTRIBUTES ) NULL,
+    (DWORD) creationDisposition,
+    dwFlagsAndAttributes,
+    NULL);
+  if (hFile == INVALID_HANDLE_VALUE) {
+    throw_ioe(env, GetLastError());
+    goto cleanup;
+  }
+
+  ret = fd_create(env, (long) hFile);
+cleanup:
+  if (path != NULL) {
+    (*env)->ReleaseStringChars(env, j_path, (const jchar*)path);
+  }
+  return (jobject) ret;
+#endif
+}
+
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_POSIX
+ * Method:    chmod
+ * Signature: (Ljava/lang/String;I)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_chmodImpl
+  (JNIEnv *env, jclass clazz, jstring j_path, jint mode)
 {
+#ifdef UNIX
   const char *path = (*env)->GetStringUTFChars(env, j_path, NULL);
   if (path == NULL) return; // JVM throws Exception for us
 
@@ -336,15 +471,30 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_chmod(
   }
 
   (*env)->ReleaseStringUTFChars(env, j_path, path);
+#endif
+
+#ifdef WINDOWS
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  LPCWSTR path = (LPCWSTR) (*env)->GetStringChars(env, j_path, NULL);
+  if (path == NULL) return; // JVM throws Exception for us
+
+  if ((dwRtnCode = ChangeFileModeByMask((LPCWSTR) path, mode)) != ERROR_SUCCESS)
+  {
+    throw_ioe(env, dwRtnCode);
+  }
+
+  (*env)->ReleaseStringChars(env, j_path, (const jchar*) path);
+#endif
 }
 
 /*
  * static native String getUserName(int uid);
  */
 JNIEXPORT jstring JNICALL 
-Java_org_apache_hadoop_io_nativeio_NativeIO_getUserName(JNIEnv *env, 
-jclass clazz, jint uid)
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getUserName(
+  JNIEnv *env, jclass clazz, jint uid)
 {
+#ifdef UNIX
   int pw_lock_locked = 0;
   if (pw_lock_object != NULL) {
     if ((*env)->MonitorEnter(env, pw_lock_object) != JNI_OK) {
@@ -396,15 +546,26 @@ cleanup:
   }
   if (pw_buf != NULL) free(pw_buf);
   return jstr_username;
+#endif // UNIX
+
+#ifdef WINDOWS
+  THROW(env, "java/io/IOException",
+    "The function POSIX.getUserName() is not supported on Windows");
+  return NULL;
+#endif
 }
 
 /*
  * static native String getGroupName(int gid);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
  */
 JNIEXPORT jstring JNICALL 
-Java_org_apache_hadoop_io_nativeio_NativeIO_getGroupName(JNIEnv *env, 
-jclass clazz, jint gid)
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getGroupName(
+  JNIEnv *env, jclass clazz, jint gid)
 {
+#ifdef UNIX
   int pw_lock_locked = 0;
  
   if (pw_lock_object != NULL) {
@@ -458,14 +619,21 @@ cleanup:
   }
   if (pw_buf != NULL) free(pw_buf);
   return jstr_groupname;
-}
+#endif  //   UNIX
 
+#ifdef WINDOWS
+  THROW(env, "java/io/IOException",
+    "The function POSIX.getUserName() is not supported on Windows");
+  return NULL;
+#endif
+}
 
 /*
  * Throw a java.IO.IOException, generating the message from errno.
  */
 static void throw_ioe(JNIEnv* env, int errnum)
 {
+#ifdef UNIX
   char message[80];
   jstring jstr_message;
 
@@ -490,9 +658,51 @@ static void throw_ioe(JNIEnv* env, int errnum)
 err:
   if (jstr_message != NULL)
     (*env)->ReleaseStringUTFChars(env, jstr_message, message);
-}
+#endif
+
+#ifdef WINDOWS
+  DWORD len = 0;
+  LPWSTR buffer = NULL;
+  const jchar* message = NULL;
+  jstring jstr_message = NULL;
+  jthrowable obj = NULL;
+
+  len = FormatMessageW(
+    FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    NULL, *(DWORD*) (&errnum), // reinterpret cast
+    MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+    (LPWSTR) &buffer, 0, NULL);
+
+  if (len > 0)
+  {
+    message = (const jchar*) buffer;
+  }
+  else
+  {
+    message = (const jchar*) L"Unknown error.";
+  }
+
+  if ((jstr_message = (*env)->NewString(env, message, len)) == NULL)
+    goto err;
+  LocalFree(buffer);
+  buffer = NULL; // Set buffer to NULL to avoid double free
+
+  obj = (jthrowable)(*env)->NewObject(env, nioe_clazz, nioe_ctor,
+    jstr_message, errnum);
+  if (obj == NULL) goto err;
+
+  (*env)->Throw(env, obj);
+  return;
 
+err:
+  if (jstr_message != NULL)
+    (*env)->ReleaseStringChars(env, jstr_message, message);
+  LocalFree(buffer);
+  return;
+#endif
+}
 
+#ifdef UNIX
 /*
  * Determine how big a buffer we need for reentrant getpwuid_r and getgrnam_r
  */
@@ -503,6 +713,104 @@ ssize_t get_pw_buflen() {
   #endif
   return (ret > 512) ? ret : 512;
 }
+#endif
+
+
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    getOwnerOnWindows
+ * Signature: (Ljava/io/FileDescriptor;)Ljava/lang/String;
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT jstring JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_getOwner
+  (JNIEnv *env, jclass clazz, jobject fd_object)
+{
+#ifdef UNIX
+  THROW(env, "java/io/IOException",
+    "The function Windows.getOwner() is not supported on Unix");
+  return NULL;
+#endif
+
+#ifdef WINDOWS
+  PSID pSidOwner = NULL;
+  PSECURITY_DESCRIPTOR pSD = NULL;
+  LPWSTR ownerName = (LPWSTR)NULL;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  jstring jstr_username = NULL;
+  HANDLE hFile = (HANDLE) fd_get(env, fd_object);
+  PASS_EXCEPTIONS_GOTO(env, cleanup);
+
+  dwRtnCode = GetSecurityInfo(
+    hFile,
+    SE_FILE_OBJECT,
+    OWNER_SECURITY_INFORMATION,
+    &pSidOwner,
+    NULL,
+    NULL,
+    NULL,
+    &pSD);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioe(env, dwRtnCode);
+    goto cleanup;
+  }
+
+  dwRtnCode = GetAccntNameFromSid(pSidOwner, &ownerName);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioe(env, dwRtnCode);
+    goto cleanup;
+  }
+
+  jstr_username = (*env)->NewString(env, ownerName, (jsize) wcslen(ownerName));
+  if (jstr_username == NULL) goto cleanup;
+
+cleanup:
+  LocalFree(ownerName);
+  LocalFree(pSD);
+  return jstr_username;
+#endif
+}
+
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    setFilePointer
+ * Signature: (Ljava/io/FileDescriptor;JJ)J
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_setFilePointer
+  (JNIEnv *env, jclass clazz, jobject fd_object, jlong distanceToMove, jlong moveMethod)
+{
+#ifdef UNIX
+  THROW(env, "java/io/IOException",
+    "The function setFilePointer(FileDescriptor) is not supported on Unix");
+  return NULL;
+#endif
+
+#ifdef WINDOWS
+  DWORD distanceToMoveLow = (DWORD) distanceToMove;
+  LONG distanceToMoveHigh = (LONG) (distanceToMove >> 32);
+  DWORD distanceMovedLow = 0;
+  HANDLE hFile = (HANDLE) fd_get(env, fd_object);
+  PASS_EXCEPTIONS_GOTO(env, cleanup);
+
+  distanceMovedLow = SetFilePointer(hFile,
+    distanceToMoveLow, &distanceToMoveHigh, (DWORD) moveMethod);
+
+  if (distanceMovedLow == INVALID_SET_FILE_POINTER) {
+     throw_ioe(env, GetLastError());
+     return -1;
+  }
+
+cleanup:
+
+  return ((jlong) distanceToMoveHigh << 32) | (jlong) distanceMovedLow;
+#endif
+}
 
 JNIEXPORT void JNICALL 
 Java_org_apache_hadoop_io_nativeio_NativeIO_renameTo0(JNIEnv *env, 

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c

@@ -14,7 +14,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
- 
+
 #include <jni.h>
 #include "file_descriptor.h"
 #include "org_apache_hadoop.h"
@@ -26,6 +26,10 @@ static jfieldID fd_descriptor;
 // the no-argument constructor
 static jmethodID fd_constructor;
 
+#ifdef WINDOWS
+// the internal field for the long handle
+static jfieldID fd_handle;
+#endif
 
 void fd_init(JNIEnv* env)
 {
@@ -37,6 +41,12 @@ void fd_init(JNIEnv* env)
 
   fd_descriptor = (*env)->GetFieldID(env, fd_class, "fd", "I");
   PASS_EXCEPTIONS(env);
+
+#ifdef WINDOWS
+  fd_handle = (*env)->GetFieldID(env, fd_class, "handle", "J");
+  PASS_EXCEPTIONS(env);
+#endif
+
   fd_constructor = (*env)->GetMethodID(env, fd_class, "<init>", "()V");
 }
 
@@ -46,9 +56,13 @@ void fd_deinit(JNIEnv *env) {
     fd_class = NULL;
   }
   fd_descriptor = NULL;
+#ifdef WINDOWS
+  fd_handle = NULL;
+#endif
   fd_constructor = NULL;
 }
 
+#ifdef UNIX
 /*
  * Given an instance 'obj' of java.io.FileDescriptor, return the
  * underlying fd, or throw if unavailable
@@ -71,4 +85,31 @@ jobject fd_create(JNIEnv *env, int fd) {
 
   (*env)->SetIntField(env, obj, fd_descriptor, fd);
   return obj;
-} 
+}
+#endif
+
+#ifdef WINDOWS
+/*
+ * Given an instance 'obj' of java.io.FileDescriptor, return the
+ * underlying fd, or throw if unavailable
+ */
+long fd_get(JNIEnv* env, jobject obj) {
+  if (obj == NULL) {
+    THROW(env, "java/lang/NullPointerException",
+          "FileDescriptor object is null");
+    return -1;
+  }
+  return (long) (*env)->GetLongField(env, obj, fd_handle);
+}
+
+/*
+ * Create a FileDescriptor object corresponding to the given int fd
+ */
+jobject fd_create(JNIEnv *env, long fd) {
+  jobject obj = (*env)->NewObject(env, fd_class, fd_constructor);
+  PASS_EXCEPTIONS_RET(env, (jobject) NULL);
+
+  (*env)->SetLongField(env, obj, fd_handle, fd);
+  return obj;
+}
+#endif

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h

@@ -18,11 +18,19 @@
 #define FILE_DESCRIPTOR_H
 
 #include <jni.h>
+#include "org_apache_hadoop.h"
 
 void fd_init(JNIEnv *env);
 void fd_deinit(JNIEnv *env);
 
+#ifdef UNIX
 int fd_get(JNIEnv* env, jobject obj);
 jobject fd_create(JNIEnv *env, int fd);
+#endif
+
+#ifdef WINDOWS
+long fd_get(JNIEnv* env, jobject obj);
+jobject fd_create(JNIEnv *env, long fd);
+#endif
 
 #endif

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c

@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <jni.h>
+#include "org_apache_hadoop.h"
+#include "org_apache_hadoop_security_JniBasedUnixGroupsMapping.h"
+
+#include <assert.h>
+#include <Windows.h>
+#include "winutils.h"
+
+static jobjectArray emptyGroups = NULL;
+
+/*
+ * Throw a java.IO.IOException, generating the message from errno.
+ */
+static void throw_ioexception(JNIEnv* env, DWORD errnum)
+{
+  DWORD len = 0;
+  LPSTR buffer = NULL;
+  const char* message = NULL;
+
+  len = FormatMessageA(
+    FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    NULL, *(DWORD*) (&errnum), // reinterpret cast
+    MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+    (LPSTR*)&buffer, 0, NULL);
+
+  if (len > 0)
+  {
+    message = buffer;
+  }
+  else
+  {
+    message = "Unknown error.";
+  }
+
+  THROW(env, "java/io/IOException", message);
+
+  LocalFree(buffer);
+
+  return;
+}
+
+JNIEXPORT jobjectArray JNICALL 
+Java_org_apache_hadoop_security_JniBasedUnixGroupsMapping_getGroupForUser 
+(JNIEnv *env, jobject jobj, jstring juser) {
+  const WCHAR *user = NULL;
+  jobjectArray jgroups = NULL;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  LPLOCALGROUP_USERS_INFO_0 groups = NULL;
+  LPLOCALGROUP_USERS_INFO_0 tmpGroups = NULL;
+  DWORD ngroups = 0;
+
+  int i;
+
+  if (emptyGroups == NULL) {
+    jobjectArray lEmptyGroups = (jobjectArray)(*env)->NewObjectArray(env, 0,
+            (*env)->FindClass(env, "java/lang/String"), NULL);
+    if (lEmptyGroups == NULL) {
+      goto cleanup;
+    }
+    emptyGroups = (*env)->NewGlobalRef(env, lEmptyGroups);
+    if (emptyGroups == NULL) {
+      goto cleanup;
+    }
+  }
+  user = (*env)->GetStringChars(env, juser, NULL);
+  if (user == NULL) {
+    THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for user buffer");
+    goto cleanup;
+  }
+
+  dwRtnCode = GetLocalGroupsForUser(user, &groups, &ngroups);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioexception(env, dwRtnCode);
+    goto cleanup;
+  }
+
+  jgroups = (jobjectArray)(*env)->NewObjectArray(env, ngroups, 
+            (*env)->FindClass(env, "java/lang/String"), NULL);
+  if (jgroups == NULL) {
+    THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for group buffer");
+    goto cleanup; 
+  }
+
+  // use a tmp pointer to iterate over groups and keep the original pointer
+  // for memory deallocation
+  tmpGroups = groups;
+
+  // fill the output string array
+  for (i = 0; i < ngroups; i++) {
+    jsize groupStringLen = (jsize)wcslen(tmpGroups->lgrui0_name);
+    jstring jgrp = (*env)->NewString(env, tmpGroups->lgrui0_name, groupStringLen);
+    if (jgrp == NULL) {
+      THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for groups buffer");
+      goto cleanup;
+    }
+    (*env)->SetObjectArrayElement(env, jgroups, i, jgrp);
+    // move on to the next group
+    tmpGroups++;
+  }
+
+cleanup:
+  if (groups != NULL) NetApiBufferFree(groups);
+
+  if (user != NULL) {
+    (*env)->ReleaseStringChars(env, juser, user);
+  }
+
+  if (dwRtnCode == ERROR_SUCCESS) {
+    return jgroups;
+  } else {
+    return emptyGroups;
+  }
+}

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c

@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
 #include "config.h"
+#endif // UNIX
 
 #include <jni.h>
 
@@ -28,4 +32,4 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
 #else
   return JNI_FALSE;
 #endif
-}
+}

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c

@@ -16,18 +16,22 @@
  * limitations under the License.
  */
 
-#include <arpa/inet.h>
+#include "org_apache_hadoop.h"
+#include "org_apache_hadoop_util_NativeCrc32.h"
+
 #include <assert.h>
-#include <inttypes.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
-#include <unistd.h>
 
+#ifdef UNIX
+#include <inttypes.h>
+#include <arpa/inet.h>
+#include <unistd.h>
 #include "config.h"
-#include "org_apache_hadoop.h"
-#include "org_apache_hadoop_util_NativeCrc32.h"
 #include "gcc_optimizations.h"
+#endif // UNIX
+
 #include "bulk_crc32.h"
 
 static void throw_checksum_exception(JNIEnv *env,
@@ -36,6 +40,9 @@ static void throw_checksum_exception(JNIEnv *env,
   char message[1024];
   jstring jstr_message;
   char *filename;
+  jclass checksum_exception_clazz;
+  jmethodID checksum_exception_ctor;
+  jthrowable obj;
 
   // Get filename as C string, or "null" if not provided
   if (j_filename == NULL) {
@@ -50,28 +57,38 @@ static void throw_checksum_exception(JNIEnv *env,
   }
 
   // Format error message
+#ifdef WINDOWS
+  _snprintf_s(
+	message,
+	sizeof(message),
+	_TRUNCATE,
+    "Checksum error: %s at %I64d exp: %d got: %d",
+    filename, pos, expected_crc, got_crc);
+#else
   snprintf(message, sizeof(message),
     "Checksum error: %s at %"PRId64" exp: %"PRId32" got: %"PRId32,
     filename, pos, expected_crc, got_crc);
+#endif // WINDOWS
+
   if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) {
     goto cleanup;
   }
  
   // Throw exception
-  jclass checksum_exception_clazz = (*env)->FindClass(
+  checksum_exception_clazz = (*env)->FindClass(
     env, "org/apache/hadoop/fs/ChecksumException");
   if (checksum_exception_clazz == NULL) {
     goto cleanup;
   }
 
-  jmethodID checksum_exception_ctor = (*env)->GetMethodID(env,
+  checksum_exception_ctor = (*env)->GetMethodID(env,
     checksum_exception_clazz, "<init>",
     "(Ljava/lang/String;J)V");
   if (checksum_exception_ctor == NULL) {
     goto cleanup;
   }
 
-  jthrowable obj = (jthrowable)(*env)->NewObject(env, checksum_exception_clazz,
+  obj = (jthrowable)(*env)->NewObject(env, checksum_exception_clazz,
     checksum_exception_ctor, jstr_message, pos);
   if (obj == NULL) goto cleanup;
 
@@ -103,6 +120,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk
     jobject j_data, jint data_offset, jint data_len,
     jstring j_filename, jlong base_pos)
 {
+  uint8_t *sums_addr;
+  uint8_t *data_addr;
+  uint32_t *sums;
+  uint8_t *data;
+  int crc_type;
+  crc32_error_t error_data;
+  int ret;
+
   if (unlikely(!j_sums || !j_data)) {
     THROW(env, "java/lang/NullPointerException",
       "input ByteBuffers must not be null");
@@ -110,8 +135,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk
   }
 
   // Convert direct byte buffers to C pointers
-  uint8_t *sums_addr = (*env)->GetDirectBufferAddress(env, j_sums);
-  uint8_t *data_addr = (*env)->GetDirectBufferAddress(env, j_data);
+  sums_addr = (*env)->GetDirectBufferAddress(env, j_sums);
+  data_addr = (*env)->GetDirectBufferAddress(env, j_data);
 
   if (unlikely(!sums_addr || !data_addr)) {
     THROW(env, "java/lang/IllegalArgumentException",
@@ -129,16 +154,15 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk
     return;
   }
 
-  uint32_t *sums = (uint32_t *)(sums_addr + sums_offset);
-  uint8_t *data = data_addr + data_offset;
+  sums = (uint32_t *)(sums_addr + sums_offset);
+  data = data_addr + data_offset;
 
   // Convert to correct internal C constant for CRC type
-  int crc_type = convert_java_crc_type(env, j_crc_type);
+  crc_type = convert_java_crc_type(env, j_crc_type);
   if (crc_type == -1) return; // exception already thrown
 
   // Setup complete. Actually verify checksums.
-  crc32_error_t error_data;
-  int ret = bulk_verify_crc(data, data_len, sums, crc_type,
+  ret = bulk_verify_crc(data, data_len, sums, crc_type,
                             bytes_per_checksum, &error_data);
   if (likely(ret == CHECKSUMS_VALID)) {
     return;

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c

@@ -21,25 +21,31 @@
  *   All rights reserved. Use of this source code is governed by a
  *   BSD-style license that can be found in the LICENSE file.
  */
+
+#include "org_apache_hadoop.h"
+
 #include <assert.h>
-#include <arpa/inet.h>
 #include <errno.h>
 #include <stdint.h>
+
+#ifdef UNIX
+#include <arpa/inet.h>
 #include <unistd.h>
+#endif // UNIX
 
 #include "crc32_zlib_polynomial_tables.h"
 #include "crc32c_tables.h"
 #include "bulk_crc32.h"
 #include "gcc_optimizations.h"
 
-#ifndef __FreeBSD__
+#if (!defined(__FreeBSD__) && !defined(WINDOWS))
 #define USE_PIPELINED
 #endif
 
 #define CRC_INITIAL_VAL 0xffffffff
 
 typedef uint32_t (*crc_update_func_t)(uint32_t, const uint8_t *, size_t);
-static inline uint32_t crc_val(uint32_t crc);
+static uint32_t crc_val(uint32_t crc);
 static uint32_t crc32_zlib_sb8(uint32_t crc, const uint8_t *buf, size_t length);
 static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length);
 
@@ -187,7 +193,7 @@ return_crc_error:
 /**
  * Extract the final result of a CRC
  */
-static inline uint32_t crc_val(uint32_t crc) {
+uint32_t crc_val(uint32_t crc) {
   return ~crc;
 }
 
@@ -200,11 +206,13 @@ static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length) {
   uint32_t end_bytes = length - running_length; 
   int li;
   for (li=0; li < running_length/8; li++) {
+	uint32_t term1;
+	uint32_t term2;
     crc ^= *(uint32_t *)buf;
     buf += 4;
-    uint32_t term1 = CRC32C_T8_7[crc & 0x000000FF] ^
+    term1 = CRC32C_T8_7[crc & 0x000000FF] ^
         CRC32C_T8_6[(crc >> 8) & 0x000000FF];
-    uint32_t term2 = crc >> 16;
+    term2 = crc >> 16;
     crc = term1 ^
         CRC32C_T8_5[term2 & 0x000000FF] ^ 
         CRC32C_T8_4[(term2 >> 8) & 0x000000FF];
@@ -234,11 +242,13 @@ static uint32_t crc32_zlib_sb8(
   uint32_t end_bytes = length - running_length; 
   int li;
   for (li=0; li < running_length/8; li++) {
+	uint32_t term1;
+	uint32_t term2;
     crc ^= *(uint32_t *)buf;
     buf += 4;
-    uint32_t term1 = CRC32_T8_7[crc & 0x000000FF] ^
+    term1 = CRC32_T8_7[crc & 0x000000FF] ^
         CRC32_T8_6[(crc >> 8) & 0x000000FF];
-    uint32_t term2 = crc >> 16;
+    term2 = crc >> 16;
     crc = term1 ^
         CRC32_T8_5[term2 & 0x000000FF] ^ 
         CRC32_T8_4[(term2 >> 8) & 0x000000FF];

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h

@@ -19,7 +19,10 @@
 #define BULK_CRC32_H_INCLUDED
 
 #include <stdint.h>
+
+#ifdef UNIX
 #include <unistd.h> /* for size_t */
+#endif // UNIX
 
 // Constants for different CRC algorithms
 #define CRC32C_POLYNOMIAL 1

hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h

@@ -17,19 +17,22 @@
  */
 
 /**
- * This file includes some common utilities 
+ * This file includes some common utilities
  * for all native code used in hadoop.
  */
- 
+
 #if !defined ORG_APACHE_HADOOP_H
 #define ORG_APACHE_HADOOP_H
 
-#include <dlfcn.h>
-#include <jni.h>
-
-#include "config.h"
+#if defined(_WIN32)
+#undef UNIX
+#define WINDOWS
+#else
+#undef WINDOWS
+#define UNIX
+#endif
 
-/* A helper macro to 'throw' a java exception. */ 
+/* A helper macro to 'throw' a java exception. */
 #define THROW(env, exception_name, message) \
   { \
 	jclass ecls = (*env)->FindClass(env, exception_name); \
@@ -55,13 +58,21 @@
     if ((*env)->ExceptionCheck(env)) return (ret); \
   }
 
-/** 
- * A helper function to dlsym a 'symbol' from a given library-handle. 
- * 
+/**
+ * Unix definitions
+ */
+#ifdef UNIX
+#include <config.h>
+#include <dlfcn.h>
+#include <jni.h>
+
+/**
+ * A helper function to dlsym a 'symbol' from a given library-handle.
+ *
  * @param env jni handle to report contingencies.
  * @param handle handle to the dlopen'ed library.
  * @param symbol symbol to load.
- * @return returns the address where the symbol is loaded in memory, 
+ * @return returns the address where the symbol is loaded in memory,
  *         <code>NULL</code> on error.
  */
 static __attribute__ ((unused))
@@ -84,6 +95,76 @@ void *do_dlsym(JNIEnv *env, void *handle, const char *symbol) {
   if ((func_ptr = do_dlsym(env, handle, symbol)) == NULL) { \
     return; \
   }
+#endif
+// Unix part end
+
+
+/**
+ * Windows definitions
+ */
+#ifdef WINDOWS
+
+/* Force using Unicode throughout the code */
+#ifndef UNICODE
+#define UNICODE
+#endif
+
+/* Microsoft C Compiler does not support the C99 inline keyword */
+#ifndef __cplusplus
+#define inline __inline;
+#endif // _cplusplus
+
+/* Optimization macros supported by GCC but for which there is no
+   direct equivalent in the Microsoft C compiler */
+#define likely(_c) (_c)
+#define unlikely(_c) (_c)
+
+/* Disable certain warnings in the native CRC32 code. */
+#pragma warning(disable:4018)		// Signed/unsigned mismatch.
+#pragma warning(disable:4244)		// Possible loss of data in conversion.
+#pragma warning(disable:4267)		// Possible loss of data.
+#pragma warning(disable:4996)		// Use of deprecated function.
+
+#include <Windows.h>
+#include <stdio.h>
+#include <jni.h>
+
+#define snprintf(a, b ,c, d) _snprintf_s((a), (b), _TRUNCATE, (c), (d))
+
+/* A helper macro to dlsym the requisite dynamic symbol and bail-out on error. */
+#define LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, env, handle, symbol) \
+  if ((func_ptr = (func_type) do_dlsym(env, handle, symbol)) == NULL) { \
+    return; \
+  }
+
+/**
+ * A helper function to dynamic load a 'symbol' from a given library-handle.
+ *
+ * @param env jni handle to report contingencies.
+ * @param handle handle to the dynamic library.
+ * @param symbol symbol to load.
+ * @return returns the address where the symbol is loaded in memory,
+ *         <code>NULL</code> on error.
+ */
+static FARPROC WINAPI do_dlsym(JNIEnv *env, HMODULE handle, LPCSTR symbol) {
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  FARPROC func_ptr = NULL;
+
+  if (!env || !handle || !symbol) {
+    THROW(env, "java/lang/InternalError", NULL);
+    return NULL;
+  }
+
+  func_ptr = GetProcAddress(handle, symbol);
+  if (func_ptr == NULL)
+  {
+    THROW(env, "java/lang/UnsatisfiedLinkError", symbol);
+  }
+  return func_ptr;
+}
+#endif
+// Windows part end
+
 
 #define LOCK_CLASS(env, clazz, classname) \
   if ((*env)->MonitorEnter(env, clazz) != 0) { \

hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c

@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+#include "org_apache_hadoop.h"
+
 #include "bulk_crc32.h"
 
 #include <stdint.h>

hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto

@@ -1,4 +1,4 @@
-/**
+/**DER
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,20 +28,17 @@ option java_generate_equals_and_hash = true;
 package hadoop.common;
 
 /**
- * This message is used for Protobuf Rpc Engine.
- * The message is used to marshal a Rpc-request
- * from RPC client to the RPC server.
+ * This message is the header for the Protobuf Rpc Engine
+ * when sending a RPC request from  RPC client to the RPC server.
+ * The actual request (serialized as protobuf) follows this request.
  *
  * No special header is needed for the Rpc Response for Protobuf Rpc Engine.
  * The normal RPC response header (see RpcHeader.proto) are sufficient. 
  */
-message RequestProto {
+message RequestHeaderProto {
   /** Name of the RPC method */
   required string methodName = 1;
 
-  /** Bytes corresponding to the client protobuf request */
-  optional bytes request = 2;
-  
   /** 
    * RPCs for a particular interface (ie protocol) are done using a
    * IPC connection that is setup using rpcProxy.

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -317,6 +317,20 @@
   are discovered using a Java ServiceLoader.</description>
 </property>
 
+<property>
+  <name>io.compression.codec.bzip2.library</name>
+  <value>system-native</value>
+  <description>The native-code library to be used for compression and
+  decompression by the bzip2 codec.  This library could be specified
+  either by by name or the full pathname.  In the former case, the
+  library is located by the dynamic linker, usually searching the
+  directories specified in the environment variable LD_LIBRARY_PATH.
+  
+  The value of "system-native" indicates that the default system
+  library should be used.  To indicate that the algorithm should
+  operate entirely in Java, specify "java-builtin".</description>
+</property>
+
 <property>
   <name>io.serializations</name>
   <value>org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>

hadoop-common-project/hadoop-common/src/main/winutils/chmod.c

@@ -0,0 +1,893 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+* http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include <errno.h>
+
+enum CHMOD_WHO
+{
+  CHMOD_WHO_NONE  =    0,
+  CHMOD_WHO_OTHER =   07,
+  CHMOD_WHO_GROUP =  070,
+  CHMOD_WHO_USER  = 0700,
+  CHMOD_WHO_ALL   = CHMOD_WHO_OTHER | CHMOD_WHO_GROUP | CHMOD_WHO_USER
+};
+
+enum CHMOD_OP
+{
+  CHMOD_OP_INVALID,
+  CHMOD_OP_PLUS,
+  CHMOD_OP_MINUS,
+  CHMOD_OP_EQUAL,
+};
+
+enum CHMOD_PERM
+{
+  CHMOD_PERM_NA =  00,
+  CHMOD_PERM_R  =  01,
+  CHMOD_PERM_W  =  02,
+  CHMOD_PERM_X  =  04,
+  CHMOD_PERM_LX = 010,
+};
+
+/*
+ * We use the following struct to build a linked list of mode change actions.
+ * The mode is described by the following grammar:
+ *  mode         ::= clause [, clause ...]
+ *  clause       ::= [who ...] [action ...]
+ *  action       ::= op [perm ...] | op [ref]
+ *  who          ::= a | u | g | o
+ *  op           ::= + | - | =
+ *  perm         ::= r | w | x | X
+ *  ref          ::= u | g | o
+ */
+typedef struct _MODE_CHANGE_ACTION
+{
+  USHORT who;
+  USHORT op;
+  USHORT perm;
+  USHORT ref;
+  struct _MODE_CHANGE_ACTION *next_action;
+} MODE_CHANGE_ACTION, *PMODE_CHANGE_ACTION;
+
+const MODE_CHANGE_ACTION INIT_MODE_CHANGE_ACTION = {
+  CHMOD_WHO_NONE, CHMOD_OP_INVALID, CHMOD_PERM_NA, CHMOD_WHO_NONE, NULL
+};
+
+static BOOL ParseOctalMode(LPCWSTR tsMask, INT *uMask);
+
+static BOOL ParseMode(LPCWSTR modeString, PMODE_CHANGE_ACTION *actions);
+
+static BOOL FreeActions(PMODE_CHANGE_ACTION actions);
+
+static BOOL ParseCommandLineArguments(__in int argc, __in wchar_t *argv[],
+  __out BOOL *rec, __out_opt INT *mask,
+  __out_opt PMODE_CHANGE_ACTION *actions, __out LPCWSTR *path);
+
+static BOOL ChangeFileModeByActions(__in LPCWSTR path,
+  PMODE_CHANGE_ACTION actions);
+
+static BOOL ChangeFileMode(__in LPCWSTR path, __in_opt INT mode,
+  __in_opt PMODE_CHANGE_ACTION actions);
+
+static BOOL ChangeFileModeRecursively(__in LPCWSTR path, __in_opt INT mode,
+  __in_opt PMODE_CHANGE_ACTION actions);
+
+
+//----------------------------------------------------------------------------
+// Function: Chmod
+//
+// Description:
+//  The main method for chmod command
+//
+// Returns:
+//  0: on success
+//
+// Notes:
+//
+int Chmod(int argc, wchar_t *argv[])
+{
+  LPWSTR pathName = NULL;
+  LPWSTR longPathName = NULL;
+
+  BOOL recursive = FALSE;
+
+  PMODE_CHANGE_ACTION actions = NULL;
+
+  INT unixAccessMask = 0;
+
+  DWORD dwRtnCode = 0;
+
+  int ret = EXIT_FAILURE;
+
+  // Parsing chmod arguments
+  //
+  if (!ParseCommandLineArguments(argc, argv,
+    &recursive, &unixAccessMask, &actions, &pathName))
+  {
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    ChmodUsage(argv[0]);
+    return EXIT_FAILURE;
+  }
+
+  // Convert the path the the long path
+  //
+  dwRtnCode = ConvertToLongPath(pathName, &longPathName);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"ConvertToLongPath", dwRtnCode);
+    goto ChmodEnd;
+  }
+
+  if (!recursive)
+  {
+    if (ChangeFileMode(longPathName, unixAccessMask, actions))
+    {
+      ret = EXIT_SUCCESS;
+    }
+  }
+  else
+  {
+    if (ChangeFileModeRecursively(longPathName, unixAccessMask, actions))
+    {
+      ret = EXIT_SUCCESS;
+    }
+  }
+
+ChmodEnd:
+  FreeActions(actions);
+  LocalFree(longPathName);
+
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileMode
+//
+// Description:
+//  Wrapper function for change file mode. Choose either change by action or by
+//  access mask.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//
+static BOOL ChangeFileMode(__in LPCWSTR path, __in_opt INT unixAccessMask,
+  __in_opt PMODE_CHANGE_ACTION actions)
+{
+  if (actions != NULL)
+    return ChangeFileModeByActions(path, actions);
+  else
+  {
+    DWORD dwRtnCode = ChangeFileModeByMask(path, unixAccessMask);
+    if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"ChangeFileModeByMask", dwRtnCode);
+      return FALSE;
+    }
+    return TRUE;
+  }
+}
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileModeRecursively
+//
+// Description:
+//  Travel the directory recursively to change the permissions.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//  The recursion works in the following way:
+//    - If the path is not a directory, change its mode and return.
+//      Symbolic links and junction points are not considered as directories.
+//    - Otherwise, call the method on all its children, then change its mode.
+//
+static BOOL ChangeFileModeRecursively(__in LPCWSTR path, __in_opt INT mode,
+  __in_opt PMODE_CHANGE_ACTION actions)
+{
+  BOOL isDir = FALSE;
+  BOOL isSymlink = FALSE;
+  LPWSTR dir = NULL;
+
+  size_t pathSize = 0;
+  size_t dirSize = 0;
+
+  HANDLE hFind = INVALID_HANDLE_VALUE;
+  WIN32_FIND_DATA ffd;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  BOOL ret = FALSE;
+
+  if ((dwRtnCode = DirectoryCheck(path, &isDir)) != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"IsDirectory", dwRtnCode);
+    return FALSE;
+  }
+  if ((dwRtnCode = SymbolicLinkCheck(path, &isSymlink)) != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"IsSymbolicLink", dwRtnCode);
+    return FALSE;
+  }
+
+  if (isSymlink || !isDir)
+  {
+     if (ChangeFileMode(path, mode, actions))
+       return TRUE;
+     else
+       return FALSE;
+  }
+
+  if (FAILED(StringCchLengthW(path, STRSAFE_MAX_CCH - 3, &pathSize)))
+  {
+    return FALSE;
+  }
+  dirSize = pathSize + 3;
+  dir = (LPWSTR)LocalAlloc(LPTR, dirSize * sizeof(WCHAR));
+  if (dir == NULL)
+  {
+    ReportErrorCode(L"LocalAlloc", GetLastError());
+    goto ChangeFileModeRecursivelyEnd;
+  }
+
+  if (FAILED(StringCchCopyW(dir, dirSize, path)) ||
+    FAILED(StringCchCatW(dir, dirSize, L"\\*")))
+  {
+    goto ChangeFileModeRecursivelyEnd;
+  }
+
+  hFind = FindFirstFile(dir, &ffd);
+  if (hFind == INVALID_HANDLE_VALUE)
+  {
+    ReportErrorCode(L"FindFirstFile", GetLastError());
+    goto ChangeFileModeRecursivelyEnd;
+  }
+
+  do
+  {
+    LPWSTR filename = NULL;
+    LPWSTR longFilename = NULL;
+    size_t filenameSize = 0;
+
+    if (wcscmp(ffd.cFileName, L".") == 0 ||
+      wcscmp(ffd.cFileName, L"..") == 0)
+      continue;
+
+    filenameSize = pathSize + wcslen(ffd.cFileName) + 2;
+    filename = (LPWSTR)LocalAlloc(LPTR, filenameSize * sizeof(WCHAR));
+    if (filename == NULL)
+    {
+      ReportErrorCode(L"LocalAlloc", GetLastError());
+      goto ChangeFileModeRecursivelyEnd;
+    }
+
+    if (FAILED(StringCchCopyW(filename, filenameSize, path)) ||
+      FAILED(StringCchCatW(filename, filenameSize, L"\\")) ||
+      FAILED(StringCchCatW(filename, filenameSize, ffd.cFileName)))
+    {
+      LocalFree(filename);
+      goto ChangeFileModeRecursivelyEnd;
+    }
+     
+    // The child fileanme is not prepended with long path prefix.
+    // Convert the filename to long path format.
+    //
+    dwRtnCode = ConvertToLongPath(filename, &longFilename);
+    LocalFree(filename);
+    if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"ConvertToLongPath", dwRtnCode);
+      LocalFree(longFilename);
+      goto ChangeFileModeRecursivelyEnd;
+    }
+
+    if(!ChangeFileModeRecursively(longFilename, mode, actions))
+    {
+      LocalFree(longFilename);
+      goto ChangeFileModeRecursivelyEnd;
+    }
+
+    LocalFree(longFilename);
+
+  } while (FindNextFileW(hFind, &ffd));
+
+  if (!ChangeFileMode(path, mode, actions))
+  {
+    goto ChangeFileModeRecursivelyEnd;
+  }
+
+  ret = TRUE;
+
+ChangeFileModeRecursivelyEnd:
+  LocalFree(dir);
+
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLineArguments
+//
+// Description:
+//  Parse command line arguments for chmod.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//	1. Recursive is only set on directories
+//  2. 'actions' is NULL if the mode is octal
+//
+static BOOL ParseCommandLineArguments(__in int argc, __in wchar_t *argv[],
+  __out BOOL *rec,
+  __out_opt INT *mask,
+  __out_opt PMODE_CHANGE_ACTION *actions,
+  __out LPCWSTR *path)
+{
+  LPCWSTR maskString;
+  BY_HANDLE_FILE_INFORMATION fileInfo;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  assert(path != NULL);
+
+  if (argc != 3 && argc != 4)
+    return FALSE;
+
+  *rec = FALSE;
+  if (argc == 4)
+  {
+    maskString = argv[2];
+    *path = argv[3];
+
+    if (wcscmp(argv[1], L"-R") == 0)
+    {
+      // Check if the given path name is a file or directory
+      // Only set recursive flag if the given path is a directory
+      //
+      dwRtnCode = GetFileInformationByName(*path, FALSE, &fileInfo);
+      if (dwRtnCode != ERROR_SUCCESS)
+      {
+        ReportErrorCode(L"GetFileInformationByName", dwRtnCode);
+        return FALSE;
+      }
+
+      if (IsDirFileInfo(&fileInfo))
+      {
+        *rec = TRUE;
+      }
+    }
+    else
+      return FALSE;
+  }
+  else
+  {
+    maskString = argv[1];
+    *path = argv[2];
+  }
+
+  if (ParseOctalMode(maskString, mask))
+  {
+    return TRUE;
+  }
+  else if (ParseMode(maskString, actions))
+  {
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
+//----------------------------------------------------------------------------
+// Function: FreeActions
+//
+// Description:
+//  Free a linked list of mode change actions given the head node.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//  none
+//
+static BOOL FreeActions(PMODE_CHANGE_ACTION actions)
+{
+  PMODE_CHANGE_ACTION curr = NULL;
+  PMODE_CHANGE_ACTION next = NULL;
+  
+  // Nothing to free if NULL is passed in
+  //
+  if (actions == NULL)
+  {
+    return TRUE;
+  }
+
+  curr = actions;
+  while (curr != NULL)
+  {
+    next = curr->next_action;
+    LocalFree(curr);
+    curr = next;
+  }
+  actions = NULL;
+
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: ComputeNewMode
+//
+// Description:
+//  Compute a new mode based on the old mode and a mode change action.
+//
+// Returns:
+//  The newly computed mode
+//
+// Notes:
+//  Apply 'rwx' permission mask or reference permission mode according to the
+//  '+', '-', or '=' operator.
+//
+static INT ComputeNewMode(__in INT oldMode,
+  __in USHORT who, __in USHORT op,
+  __in USHORT perm, __in USHORT ref)
+{
+  static const INT readMask  = 0444;
+  static const INT writeMask = 0222;
+  static const INT exeMask   = 0111;
+
+  INT mask = 0;
+  INT mode = 0;
+
+  // Operations are exclusive, and cannot be invalid
+  //
+  assert(op == CHMOD_OP_EQUAL || op == CHMOD_OP_PLUS || op == CHMOD_OP_MINUS);
+
+  // Nothing needs to be changed if there is not permission or reference
+  //
+  if(perm == CHMOD_PERM_NA && ref == CHMOD_WHO_NONE)
+  {
+    return oldMode;
+  }
+
+  // We should have only permissions or a reference target, not both.
+  //
+  assert((perm != CHMOD_PERM_NA && ref == CHMOD_WHO_NONE) ||
+    (perm == CHMOD_PERM_NA && ref != CHMOD_WHO_NONE));
+
+  if (perm != CHMOD_PERM_NA)
+  {
+    if ((perm & CHMOD_PERM_R) == CHMOD_PERM_R)
+      mask |= readMask;
+    if ((perm & CHMOD_PERM_W) == CHMOD_PERM_W)
+      mask |= writeMask;
+    if ((perm & CHMOD_PERM_X) == CHMOD_PERM_X)
+      mask |= exeMask;
+    if (((perm & CHMOD_PERM_LX) == CHMOD_PERM_LX))
+    {
+      // It applies execute permissions to directories regardless of their
+      // current permissions and applies execute permissions to a file which
+      // already has at least 1 execute permission bit already set (either user,
+      // group or other). It is only really useful when used with '+' and
+      // usually in combination with the -R option for giving group or other
+      // access to a big directory tree without setting execute permission on
+      // normal files (such as text files), which would normally happen if you
+      // just used "chmod -R a+rx .", whereas with 'X' you can do
+      // "chmod -R a+rX ." instead (Source: Wikipedia)
+      //
+      if ((oldMode & UX_DIRECTORY) == UX_DIRECTORY || (oldMode & exeMask))
+        mask |= exeMask;
+    }
+  }
+  else if (ref != CHMOD_WHO_NONE)
+  {
+    mask |= oldMode & ref;
+    switch(ref)
+    {
+    case CHMOD_WHO_GROUP:
+      mask |= mask >> 3;
+      mask |= mask << 3;
+      break;
+    case CHMOD_WHO_OTHER:
+      mask |= mask << 3;
+      mask |= mask << 6;
+      break;
+    case CHMOD_WHO_USER:
+      mask |= mask >> 3;
+      mask |= mask >> 6;
+      break;
+    default:
+      // Reference modes can only be U/G/O and are exclusive
+      assert(FALSE);
+    }
+  }
+
+  mask &= who;
+
+  if (op == CHMOD_OP_EQUAL)
+  {
+    mode = (oldMode & (~who)) | mask;
+  }
+  else if (op == CHMOD_OP_MINUS)
+  {
+    mode = oldMode & (~mask);
+  }
+  else if (op == CHMOD_OP_PLUS)
+  {
+    mode = oldMode | mask;
+  }
+
+  return mode;
+}
+
+//----------------------------------------------------------------------------
+// Function: ConvertActionsToMask
+//
+// Description:
+//  Convert a linked list of mode change actions to the Unix permission mask
+//  given the head node.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//  none
+//
+static BOOL ConvertActionsToMask(__in LPCWSTR path,
+  __in PMODE_CHANGE_ACTION actions, __out PINT puMask)
+{
+  PMODE_CHANGE_ACTION curr = NULL;
+
+  BY_HANDLE_FILE_INFORMATION fileInformation;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  INT mode = 0;
+
+  dwErrorCode = GetFileInformationByName(path, FALSE, &fileInformation);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"GetFileInformationByName", dwErrorCode);
+    return FALSE;
+  }
+  if (IsDirFileInfo(&fileInformation))
+  {
+    mode |= UX_DIRECTORY;
+  }
+  dwErrorCode = FindFileOwnerAndPermission(path, NULL, NULL, &mode);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"FindFileOwnerAndPermission", dwErrorCode);
+    return FALSE;
+  }
+  *puMask = mode;
+
+  // Nothing to change if NULL is passed in
+  //
+  if (actions == NULL)
+  {
+    return TRUE;
+  }
+
+  for (curr = actions; curr != NULL; curr = curr->next_action)
+  {
+    mode = ComputeNewMode(mode, curr->who, curr->op, curr->perm, curr->ref);
+  }
+
+  *puMask = mode;
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileModeByActions
+//
+// Description:
+//  Change a file mode through a list of actions.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//  none
+//
+static BOOL ChangeFileModeByActions(__in LPCWSTR path,
+  PMODE_CHANGE_ACTION actions)
+{
+  INT mask = 0;
+
+  if (ConvertActionsToMask(path, actions, &mask))
+  {
+    DWORD dwRtnCode = ChangeFileModeByMask(path, mask);
+    if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"ChangeFileModeByMask", dwRtnCode);
+      return FALSE;
+    }
+    return TRUE;
+  }
+  else
+    return FALSE;
+}
+
+//----------------------------------------------------------------------------
+// Function: ParseMode
+//
+// Description:
+//  Convert a mode string into a linked list of actions
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//  Take a state machine approach to parse the mode. Each mode change action
+//  will be a node in the output linked list. The state machine has five state,
+//  and each will only transit to the next; the end state can transit back to
+//  the first state, and thus form a circle. In each state, if we see a
+//  a character not belongs to the state, we will move to next state. WHO, PERM,
+//  and REF states are optional; OP and END states are required; and errors
+//  will only be reported at the latter two states.
+//
+static BOOL ParseMode(LPCWSTR modeString, PMODE_CHANGE_ACTION *pActions)
+{
+  enum __PARSE_MODE_ACTION_STATE
+  {
+    PARSE_MODE_ACTION_WHO_STATE,
+    PARSE_MODE_ACTION_OP_STATE,
+    PARSE_MODE_ACTION_PERM_STATE,
+    PARSE_MODE_ACTION_REF_STATE,
+    PARSE_MODE_ACTION_END_STATE
+  } state = PARSE_MODE_ACTION_WHO_STATE;
+
+  MODE_CHANGE_ACTION action = INIT_MODE_CHANGE_ACTION;
+  PMODE_CHANGE_ACTION actionsEnd = NULL;
+  PMODE_CHANGE_ACTION actionsLast = NULL;
+  USHORT lastWho;
+  WCHAR c = 0;
+  size_t len = 0;
+  size_t i = 0;
+
+  assert(modeString != NULL && pActions != NULL);
+
+  if (FAILED(StringCchLengthW(modeString, STRSAFE_MAX_CCH, &len)))
+  {
+    return FALSE;
+  }
+
+  actionsEnd = *pActions;
+  while(i <= len)
+  {
+    c = modeString[i];
+    if (state == PARSE_MODE_ACTION_WHO_STATE)
+    {
+      switch (c)
+      {
+      case L'a':
+        action.who |= CHMOD_WHO_ALL;
+        i++;
+        break;
+      case L'u':
+        action.who |= CHMOD_WHO_USER;
+        i++;
+        break;
+      case L'g':
+        action.who |= CHMOD_WHO_GROUP;
+        i++;
+        break;
+      case L'o':
+        action.who |= CHMOD_WHO_OTHER;
+        i++;
+        break;
+      default:
+        state = PARSE_MODE_ACTION_OP_STATE;
+      } // WHO switch
+    }
+    else if (state == PARSE_MODE_ACTION_OP_STATE)
+    {
+      switch (c)
+      {
+      case L'+':
+        action.op = CHMOD_OP_PLUS;
+        break;
+      case L'-':
+        action.op = CHMOD_OP_MINUS;
+        break;
+      case L'=':
+        action.op = CHMOD_OP_EQUAL;
+        break;
+      default:
+        fwprintf(stderr, L"Invalid mode: '%s'\n", modeString);
+        FreeActions(*pActions);
+        return FALSE;
+      } // OP switch
+      i++;
+      state = PARSE_MODE_ACTION_PERM_STATE;
+    }
+    else if (state == PARSE_MODE_ACTION_PERM_STATE)
+    {
+      switch (c)
+      {
+      case L'r':
+        action.perm |= CHMOD_PERM_R;
+        i++;
+        break;
+      case L'w':
+        action.perm |= CHMOD_PERM_W;
+        i++;
+        break;
+      case L'x':
+        action.perm |= CHMOD_PERM_X;
+        i++;
+        break;
+      case L'X':
+        action.perm |= CHMOD_PERM_LX;
+        i++;
+        break;
+      default:
+        state = PARSE_MODE_ACTION_REF_STATE;
+      } // PERM switch
+    }
+    else if (state == PARSE_MODE_ACTION_REF_STATE)
+    {
+      switch (c)
+      {
+      case L'u':
+        action.ref = CHMOD_WHO_USER;
+        i++;
+        break;
+      case L'g':
+        action.ref = CHMOD_WHO_GROUP;
+        i++;
+        break;
+      case L'o':
+        action.ref = CHMOD_WHO_OTHER;
+        i++;
+        break;
+      default:
+        state = PARSE_MODE_ACTION_END_STATE;
+      } // REF switch
+    }
+    else if (state == PARSE_MODE_ACTION_END_STATE)
+    {
+      switch (c)
+      {
+      case NULL:
+      case L',':
+        i++;
+      case L'+':
+      case L'-':
+      case L'=':
+        state = PARSE_MODE_ACTION_WHO_STATE;
+
+        // Append the current action to the end of the linked list
+        //
+        assert(actionsEnd == NULL);
+        // Allocate memory
+        actionsEnd = (PMODE_CHANGE_ACTION) LocalAlloc(LPTR,
+          sizeof(MODE_CHANGE_ACTION));
+        if (actionsEnd == NULL)
+        {
+          ReportErrorCode(L"LocalAlloc", GetLastError());
+          FreeActions(*pActions);
+          return FALSE;
+        }
+        if (action.who == CHMOD_WHO_NONE) action.who = CHMOD_WHO_ALL;
+        // Copy the action to the new node
+        *actionsEnd = action;
+        // Append to the last node in the linked list
+        if (actionsLast != NULL) actionsLast->next_action = actionsEnd;
+        // pActions should point to the head of the linked list
+        if (*pActions == NULL) *pActions = actionsEnd;
+        // Update the two pointers to point to the last node and the tail
+        actionsLast = actionsEnd;
+        actionsEnd = actionsLast->next_action;
+
+        // Reset action
+        //
+        lastWho = action.who;
+        action = INIT_MODE_CHANGE_ACTION;
+        if (c != L',')
+        {
+          action.who = lastWho;
+        }
+
+        break;
+      default:
+        fwprintf(stderr, L"Invalid mode: '%s'\n", modeString);
+        FreeActions(*pActions);
+        return FALSE;
+      } // END switch
+    }
+  } // while
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: ParseOctalMode
+//
+// Description:
+//  Convert the 3 or 4 digits Unix mask string into the binary representation
+//  of the Unix access mask, i.e. 9 bits each an indicator of the permission
+//  of 'rwxrwxrwx', i.e. user's, group's, and owner's read, write, and
+//  execute/search permissions.
+//
+// Returns:
+//  TRUE: on success
+//  FALSE: otherwise
+//
+// Notes:
+//	none
+//
+static BOOL ParseOctalMode(LPCWSTR tsMask, INT *uMask)
+{
+  size_t tsMaskLen = 0;
+  DWORD i;
+  LONG l;
+  WCHAR *end;
+
+  if (uMask == NULL)
+    return FALSE;
+
+  if (FAILED(StringCchLengthW(tsMask, STRSAFE_MAX_CCH, &tsMaskLen)))
+    return FALSE;
+
+  if (tsMaskLen == 0 || tsMaskLen > 4)
+  {
+    return FALSE;
+  }
+
+  for (i = 0; i < tsMaskLen; i++)
+  {
+    if (!(tsMask[tsMaskLen - i - 1] >= L'0' &&
+      tsMask[tsMaskLen - i - 1] <= L'7'))
+      return FALSE;
+  }
+
+  errno = 0;
+  if (tsMaskLen == 4)
+    // Windows does not have any equivalent of setuid/setgid and sticky bit.
+    // So the first bit is omitted for the 4 digit octal mode case.
+    //
+    l = wcstol(tsMask + 1, &end, 8);
+  else
+    l = wcstol(tsMask, &end, 8);
+
+  if (errno || l > 0x0777 || l < 0 || *end != 0)
+  {
+    return FALSE;
+  }
+
+  *uMask = (INT) l;
+
+  return TRUE;
+}
+
+void ChmodUsage(LPCWSTR program)
+{
+  fwprintf(stdout, L"\
+Usage: %s [OPTION] OCTAL-MODE [FILE]\n\
+   or: %s [OPTION] MODE [FILE]\n\
+Change the mode of the FILE to MODE.\n\
+\n\
+   -R: change files and directories recursively\n\
+\n\
+Each MODE is of the form '[ugoa]*([-+=]([rwxX]*|[ugo]))+'.\n",
+program, program);
+}

hadoop-common-project/hadoop-common/src/main/winutils/chown.c

@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#include "winutils.h"
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileOwnerBySid
+//
+// Description:
+//  Change a file or directory ownership by giving new owner and group SIDs
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+// Notes:
+//  This function is long path safe, i.e. the path will be converted to long
+//  path format if not already converted. So the caller does not need to do
+//  the converstion before calling the method.
+//
+static DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+  __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
+{
+  LPWSTR longPathName = NULL;
+  INT oldMode = 0;
+
+  SECURITY_INFORMATION securityInformation = 0;
+
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  // Convert the path the the long path
+  //
+  dwRtnCode = ConvertToLongPath(path, &longPathName);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto ChangeFileOwnerByNameEnd;
+  }
+
+  // Get a pointer to the existing owner information and DACL
+  //
+  dwRtnCode = FindFileOwnerAndPermission(longPathName, NULL, NULL, &oldMode);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto ChangeFileOwnerByNameEnd;
+  }
+
+  // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
+  // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
+  // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
+  // permission enabled.
+  //
+  if (!EnablePrivilege(L"SeTakeOwnershipPrivilege"))
+  {
+    fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
+  }
+  if (!EnablePrivilege(L"SeRestorePrivilege"))
+  {
+    fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
+  }
+
+  assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
+
+  // Set the owners of the file.
+  //
+  if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
+  if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
+  dwRtnCode = SetNamedSecurityInfoW(
+    longPathName,
+    SE_FILE_OBJECT,
+    securityInformation,
+    pNewOwnerSid,
+    pNewGroupSid,
+    NULL,
+    NULL);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto ChangeFileOwnerByNameEnd;
+  }
+
+  // Set the permission on the file for the new owner.
+  //
+  dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    goto ChangeFileOwnerByNameEnd;
+  }
+
+ChangeFileOwnerByNameEnd:
+  LocalFree(longPathName);
+  return dwRtnCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: Chown
+//
+// Description:
+//	The main method for chown command
+//
+// Returns:
+//	0: on success
+//
+// Notes:
+//
+//
+int Chown(int argc, wchar_t *argv[])
+{
+  LPWSTR pathName = NULL;
+
+  LPWSTR ownerInfo = NULL;
+
+  LPWSTR colonPos = NULL;
+
+  LPWSTR userName = NULL;
+  size_t userNameLen = 0;
+
+  LPWSTR groupName = NULL;
+  size_t groupNameLen = 0;
+
+  PSID pNewOwnerSid = NULL;
+  PSID pNewGroupSid = NULL;
+
+  DWORD dwRtnCode = 0;
+
+  int ret = EXIT_FAILURE;
+
+  if (argc >= 3)
+  {
+    ownerInfo = argv[1];
+    pathName = argv[2];
+  }
+  else
+  {
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    ChownUsage(argv[0]);
+    return ret;
+  }
+
+  // Parsing the owner name
+  //
+  if ((colonPos = wcschr(ownerInfo, L':')) != NULL)
+  {
+    if (colonPos - ownerInfo != 0)
+    {
+      // Length includes NULL terminator
+      userNameLen = colonPos - ownerInfo + 1;
+      userName = (LPTSTR)LocalAlloc(LPTR, userNameLen * sizeof(WCHAR));
+      if (userName == NULL)
+      {
+        ReportErrorCode(L"LocalAlloc", GetLastError());
+        goto ChownEnd;
+      }
+      if (FAILED(StringCchCopyNW(userName, userNameLen,
+        ownerInfo, userNameLen - 1)))
+        goto ChownEnd;
+    }
+
+    if (*(colonPos + 1) != 0)
+    {
+      // Length includes NULL terminator
+      groupNameLen = wcslen(ownerInfo) - (colonPos - ownerInfo) + 1;
+      groupName = (LPTSTR)LocalAlloc(LPTR, groupNameLen * sizeof(WCHAR));
+      if (groupName == NULL)
+      {
+        ReportErrorCode(L"LocalAlloc", GetLastError());
+        goto ChownEnd;
+      }
+      if (FAILED(StringCchCopyNW(groupName, groupNameLen,
+        colonPos + 1, groupNameLen)))
+        goto ChownEnd;
+    }
+  }
+  else
+  {
+    // Length includes NULL terminator
+    userNameLen = wcslen(ownerInfo) + 1;
+    userName = (LPWSTR)LocalAlloc(LPTR, userNameLen * sizeof(WCHAR));
+    if (userName == NULL)
+    {
+      ReportErrorCode(L"LocalAlloc", GetLastError());
+      goto ChownEnd;
+    }
+    if (FAILED(StringCchCopyNW(userName, userNameLen, ownerInfo, userNameLen)))
+      goto ChownEnd;
+  }
+
+  // Not allow zero length user name or group name in the parsing results.
+  //
+  assert(userName == NULL || wcslen(userName) > 0);
+  assert(groupName == NULL || wcslen(groupName) > 0);
+
+  // Nothing to change if both names are empty
+  //
+  if ((userName == NULL) && (groupName == NULL))
+  {
+    ret = EXIT_SUCCESS;
+    goto ChownEnd;
+  }
+
+  if (userName != NULL)
+  {
+    dwRtnCode = GetSidFromAcctNameW(userName, &pNewOwnerSid);
+    if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
+      fwprintf(stderr, L"Invalid user name: %s\n", userName);
+      goto ChownEnd;
+    }
+  }
+
+  if (groupName != NULL)
+  {
+    dwRtnCode = GetSidFromAcctNameW(groupName, &pNewGroupSid);
+    if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
+      fwprintf(stderr, L"Invalid group name: %s\n", groupName);
+      goto ChownEnd;
+    }
+  }
+
+  if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
+  {
+    fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
+    goto ChownEnd;
+  }
+
+  dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode);
+    goto ChownEnd;
+  }
+
+  ret = EXIT_SUCCESS;
+
+ChownEnd:
+  LocalFree(userName);
+  LocalFree(groupName);
+  LocalFree(pNewOwnerSid);
+  LocalFree(pNewGroupSid);
+
+  return ret;
+}
+
+void ChownUsage(LPCWSTR program)
+{
+  fwprintf(stdout, L"\
+Usage: %s [OWNER][:[GROUP]] [FILE]\n\
+Change the owner and/or group of the FILE to OWNER and/or GROUP.\n\
+\n\
+Note:\n\
+On Linux, if a colon but no group name follows the user name, the group of\n\
+the files is changed to that user\'s login group. Windows has no concept of\n\
+a user's login group. So we do not change the group owner in this case.\n",
+program);
+}

hadoop-common-project/hadoop-common/src/main/winutils/groups.c

@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+#include "winutils.h"
+
+//----------------------------------------------------------------------------
+// Function: PrintGroups
+//
+// Description:
+//	Print group names to the console standard output for the given user
+//
+// Returns:
+//	TRUE: on success
+//
+// Notes:
+//   This function could fail on first pass when we fail to find groups for
+//   domain account; so we do not report Windows API errors in this function.
+//   If formatOutput is true, pipe character is used as separator for groups
+//   otherwise, space.
+//
+static BOOL PrintGroups(
+  LPLOCALGROUP_USERS_INFO_0 groups,
+  DWORD entries,
+  BOOL formatOutput)
+{
+  BOOL ret = TRUE;
+  LPLOCALGROUP_USERS_INFO_0 pTmpBuf = groups;
+  DWORD i;
+
+  for (i = 0; i < entries; i++)
+  {
+    if (pTmpBuf == NULL)
+    {
+      ret = FALSE;
+      break;
+    }
+
+    if (i != 0)
+    {
+      if (formatOutput)
+      {
+        wprintf(L"|");
+      }
+      else
+      {
+        wprintf(L" ");
+      }
+    }
+    wprintf(L"%s", pTmpBuf->lgrui0_name);
+
+    pTmpBuf++;
+  }
+
+  if (ret)
+    wprintf(L"\n");
+
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//   Parses the command line
+//
+// Returns:
+//   TRUE on the valid command line, FALSE otherwise
+//
+static BOOL ParseCommandLine(
+  int argc, wchar_t *argv[], wchar_t **user, BOOL *formatOutput)
+{
+  *formatOutput = FALSE;
+
+  assert(argv != NULL);
+  assert(user != NULL);
+
+  if (argc == 1)
+  {
+    // implicitly use the current user
+    *user = NULL;
+    return TRUE;
+  }
+  else if (argc == 2)
+  {
+    // check if the second argument is formating
+    if (wcscmp(argv[1], L"-F") == 0)
+    {
+      *user = NULL;
+      *formatOutput = TRUE;
+      return TRUE;
+    }
+    else
+    {
+      *user = argv[1];
+      return TRUE;
+    }
+  }
+  else if (argc == 3 && wcscmp(argv[1], L"-F") == 0)
+  {
+    // if 3 args, the second argument must be "-F"
+
+    *user = argv[2];
+    *formatOutput = TRUE;
+    return TRUE;
+  }
+
+  return FALSE;
+}
+
+//----------------------------------------------------------------------------
+// Function: Groups
+//
+// Description:
+//	The main method for groups command
+//
+// Returns:
+//	0: on success
+//
+// Notes:
+//
+//
+int Groups(int argc, wchar_t *argv[])
+{
+  LPWSTR input = NULL;
+
+  LPWSTR currentUser = NULL;
+  DWORD cchCurrentUser = 0;
+
+  LPLOCALGROUP_USERS_INFO_0 groups = NULL;
+  DWORD entries = 0;
+
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  int ret = EXIT_SUCCESS;
+  BOOL formatOutput = FALSE;
+
+  if (!ParseCommandLine(argc, argv, &input, &formatOutput))
+  {
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    GroupsUsage(argv[0]);
+    return EXIT_FAILURE;
+  }
+
+  // if username was not specified on the command line, fallback to the
+  // current user
+  if (input == NULL)
+  {
+    GetUserNameW(currentUser, &cchCurrentUser);
+    if (GetLastError() == ERROR_INSUFFICIENT_BUFFER)
+    {
+      currentUser = (LPWSTR) LocalAlloc(LPTR,
+        (cchCurrentUser + 1) * sizeof(wchar_t));
+      if (!currentUser)
+      {
+        ReportErrorCode(L"LocalAlloc", GetLastError());
+        ret = EXIT_FAILURE;
+        goto GroupsEnd;
+      }
+      if (GetUserNameW(currentUser, &cchCurrentUser))
+        input = currentUser;
+      else
+      {
+        ReportErrorCode(L"GetUserName", GetLastError());
+        ret = EXIT_FAILURE;
+        goto GroupsEnd;
+      }
+    }
+    else
+    {
+      ReportErrorCode(L"GetUserName", GetLastError());
+      ret = EXIT_FAILURE;
+      goto GroupsEnd;
+    }
+  }
+
+  if ((dwRtnCode = GetLocalGroupsForUser(input, &groups, &entries))
+    != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"GetLocalGroupsForUser", dwRtnCode);
+    ret = EXIT_FAILURE;
+    goto GroupsEnd;
+  }
+
+  if (!PrintGroups(groups, entries, formatOutput))
+  {
+    ret = EXIT_FAILURE;
+  }
+
+GroupsEnd:
+  LocalFree(currentUser);
+  if (groups != NULL) NetApiBufferFree(groups);
+  return ret;
+}
+
+void GroupsUsage(LPCWSTR program)
+{
+  fwprintf(stdout, L"\
+Usage: %s [OPTIONS] [USERNAME]\n\
+Print group information of the specified USERNAME \
+(the current user by default).\n\
+\n\
+OPTIONS: -F format the output by separating tokens with a pipe\n",
+program);
+}

hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c

@@ -0,0 +1,230 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+* http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+
+// List of different hardlink related command line options supported by
+// winutils.
+typedef enum HardLinkCommandOptionType
+{
+  HardLinkInvalid,
+  HardLinkCreate,
+  HardLinkStat
+} HardLinkCommandOption;
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//  Parses the given command line. On success, out param 'command' contains
+//  the user specified command.
+//
+// Returns:
+// TRUE: If the command line is valid
+// FALSE: otherwise
+static BOOL ParseCommandLine(__in int argc,
+                             __in wchar_t *argv[],
+                             __out HardLinkCommandOption *command)
+{
+  *command = HardLinkInvalid;
+
+  if (argc != 3 && argc != 4) {
+    return FALSE;
+  }
+
+  if (argc == 3) {
+    if (wcscmp(argv[0], L"hardlink") != 0 || wcscmp(argv[1], L"stat") != 0)
+    {
+      return FALSE;
+    }
+
+    *command = HardLinkStat;
+  }
+
+  if (argc == 4) {
+    if (wcscmp(argv[0], L"hardlink") != 0 || wcscmp(argv[1], L"create") != 0)
+    {
+      return FALSE;
+    }
+
+    *command = HardLinkCreate;
+  }
+
+  assert(*command != HardLinkInvalid);
+
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: HardlinkStat
+//
+// Description:
+//  Computes the number of hard links for a given file.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// error code: otherwise
+static DWORD HardlinkStat(__in LPCWSTR fileName, __out DWORD *puHardLinkCount)
+{
+  BY_HANDLE_FILE_INFORMATION fileInformation;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  PWSTR longFileName = NULL;
+
+  // First convert input paths to long paths
+  //
+  dwErrorCode = ConvertToLongPath(fileName, &longFileName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    goto HardlinkStatExit;
+  }
+
+  // Get file information which contains the hard link count
+  //
+  dwErrorCode = GetFileInformationByName(longFileName, FALSE, &fileInformation);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    goto HardlinkStatExit;
+  }
+
+  *puHardLinkCount = fileInformation.nNumberOfLinks;
+
+HardlinkStatExit:
+  LocalFree(longFileName);
+
+  return dwErrorCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: HardlinkCreate
+//
+// Description:
+//  Creates a hard link for a given file under the given name.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// error code: otherwise
+static DWORD HardlinkCreate(__in LPCWSTR linkName, __in LPCWSTR fileName)
+{
+  PWSTR longLinkName = NULL;
+  PWSTR longFileName = NULL;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  // First convert input paths to long paths
+  //
+  dwErrorCode = ConvertToLongPath(linkName, &longLinkName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    goto HardlinkCreateExit;
+  }
+
+  dwErrorCode = ConvertToLongPath(fileName, &longFileName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    goto HardlinkCreateExit;
+  }
+
+  // Create the hard link
+  //
+  if (!CreateHardLink(longLinkName, longFileName, NULL))
+  {
+    dwErrorCode = GetLastError();
+  }
+
+HardlinkCreateExit:
+  LocalFree(longLinkName);
+  LocalFree(longFileName);
+
+  return dwErrorCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: Hardlink
+//
+// Description:
+//  Creates a hard link for a given file under the given name. Outputs the
+//  appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// EXIT_SUCCESS: On success
+// EXIT_FAILURE: otherwise
+int Hardlink(int argc, wchar_t *argv[])
+{
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  int ret = EXIT_FAILURE;
+  HardLinkCommandOption command = HardLinkInvalid;
+
+  if (!ParseCommandLine(argc, argv, &command)) {
+    dwErrorCode = ERROR_INVALID_COMMAND_LINE;
+
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    HardlinkUsage();
+    goto HardLinkExit;
+  }
+
+  if (command == HardLinkStat)
+  {
+    // Compute the number of hard links
+    //
+    DWORD uHardLinkCount = 0;
+    dwErrorCode = HardlinkStat(argv[2], &uHardLinkCount);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"HardlinkStat", dwErrorCode);
+      goto HardLinkExit;
+    }
+
+    // Output the result
+    //
+    fwprintf(stdout, L"%d\n", uHardLinkCount);
+
+  } else if (command == HardLinkCreate)
+  {
+    // Create the hard link
+    //
+    dwErrorCode = HardlinkCreate(argv[2], argv[3]);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"HardlinkCreate", dwErrorCode);
+      goto HardLinkExit;
+    }
+
+    // Output the success message
+    //
+    fwprintf(stdout, L"Hardlink created for %s <<===>> %s\n", argv[2], argv[3]);
+
+  } else
+  {
+    // Should not happen
+    //
+    assert(FALSE);
+  }
+
+  ret = EXIT_SUCCESS;
+
+HardLinkExit:
+
+  return ret;
+}
+
+void HardlinkUsage()
+{
+    fwprintf(stdout, L"\
+Usage: hardlink create [LINKNAME] [FILENAME] |\n\
+       hardlink stat [FILENAME]\n\
+Creates a new hardlink on the existing file or displays the number of links\n\
+for the given file\n");
+}

hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h

@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#ifndef UNICODE
+#define UNICODE
+#endif
+
+#pragma once
+
+#include <stdio.h>
+#include <assert.h>
+#include <windows.h>
+#include <aclapi.h>
+#include <accctrl.h>
+#include <tchar.h>
+#include <strsafe.h>
+#include <lm.h>
+
+enum EXIT_CODE
+{
+  /* Common success exit code shared among all utilities */
+  SUCCESS = EXIT_SUCCESS,
+  /* Generic failure exit code share among all utilities */
+  FAILURE = EXIT_FAILURE,
+  /* Failure code indicates the user does not privilege to create symlinks */
+  SYMLINK_NO_PRIVILEGE = 2,
+};
+
+
+/*
+ * The array of 12 months' three-letter abbreviations 
+ */
+extern const LPCWSTR MONTHS[];
+
+/*
+ * The Unix masks
+ * The Windows version of <sys/stat.h> does not contain all the POSIX flag/mask
+ * definitions. The following masks are used in 'winutils' to represent POSIX
+ * permission mode.
+ * 
+ */
+enum UnixAclMask
+{
+  UX_O_EXECUTE = 00001, // S_IXOTH
+  UX_O_WRITE   = 00002, // S_IWOTH
+  UX_O_READ    = 00004, // S_IROTH
+  UX_G_EXECUTE = 00010, // S_IXGRP
+  UX_G_WRITE   = 00020, // S_IWGRP
+  UX_G_READ    = 00040, // S_IRGRP
+  UX_U_EXECUTE = 00100, // S_IXUSR
+  UX_U_WRITE   = 00200, // S_IWUSR
+  UX_U_READ    = 00400, // S_IRUSR
+  UX_DIRECTORY = 0040000, // S_IFDIR
+  UX_SYMLINK   = 0120000, // S_IFLNK
+};
+
+
+/*
+ * The WindowsAclMask and WinMasks contain the definitions used to establish
+ * the mapping between Unix and Windows.
+ */
+enum WindowsAclMask
+{
+  WIN_READ, // The permission(s) that enable Unix read permission
+  WIN_WRITE, // The permission(s) that enable Unix write permission
+  WIN_EXECUTE, // The permission(s) that enbale Unix execute permission
+  WIN_OWNER_SE, // The permissions that are always set for file owners 
+  WIN_ALL, // The permissions that all files on Windows should have
+  WIN_MASKS_TOTAL
+};
+extern const ACCESS_MASK WinMasks[];
+
+
+int Ls(int argc, wchar_t *argv[]);
+void LsUsage(LPCWSTR program);
+
+int Chmod(int argc, wchar_t *argv[]);
+void ChmodUsage(LPCWSTR program);
+
+int Chown(int argc, wchar_t *argv[]);
+void ChownUsage(LPCWSTR program);
+
+int Groups(int argc, wchar_t *argv[]);
+void GroupsUsage(LPCWSTR program);
+
+int Hardlink(int argc, wchar_t *argv[]);
+void HardlinkUsage();
+
+int Task(int argc, wchar_t *argv[]);
+void TaskUsage();
+
+int Symlink(int argc, wchar_t *argv[]);
+void SymlinkUsage();
+
+int SystemInfo();
+void SystemInfoUsage();
+
+DWORD GetFileInformationByName(__in LPCWSTR pathName,  __in BOOL followLink,
+  __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
+
+DWORD ConvertToLongPath(__in PCWSTR path, __deref_out PWSTR *newPath);
+
+DWORD GetSidFromAcctNameW(LPCWSTR acctName, PSID* ppSid);
+
+DWORD GetAccntNameFromSid(PSID pSid, LPWSTR *ppAcctName);
+
+void ReportErrorCode(LPCWSTR func, DWORD err);
+
+BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation);
+
+DWORD FindFileOwnerAndPermission(
+  __in LPCWSTR pathName,
+  __out_opt LPWSTR *pOwnerName,
+  __out_opt LPWSTR *pGroupName,
+  __out_opt PINT pMask);
+
+DWORD DirectoryCheck(__in LPCWSTR pathName, __out LPBOOL result);
+
+DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out LPBOOL result);
+
+DWORD JunctionPointCheck(__in LPCWSTR pathName, __out LPBOOL result);
+
+DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode);
+
+DWORD GetLocalGroupsForUser(__in LPCWSTR user,
+  __out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries);
+
+BOOL EnablePrivilege(__in LPCWSTR privilegeName);

hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c

@@ -0,0 +1,1515 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#pragma comment(lib, "authz.lib")
+#pragma comment(lib, "netapi32.lib")
+#include "winutils.h"
+#include <authz.h>
+#include <sddl.h>
+
+/*
+ * The array of 12 months' three-letter abbreviations 
+ */
+const LPCWSTR MONTHS[] = { L"Jan", L"Feb", L"Mar", L"Apr", L"May", L"Jun",
+  L"Jul", L"Aug", L"Sep", L"Oct", L"Nov", L"Dec" };
+
+/*
+ * The WindowsAclMask and WinMasks contain the definitions used to establish
+ * the mapping between Unix and Windows.
+ * We set up the mapping with the following rules. 
+ *   1. Everyone will have WIN_ALL permissions;
+ *   2. Owner will always have WIN_OWNER_SE permissions in addition;
+ *   2. When Unix read/write/excute permission is set on the file, the
+ *      corresponding Windows allow ACE will be added to the file.
+ * More details and explaination can be found in the following white paper:
+ *   http://technet.microsoft.com/en-us/library/bb463216.aspx
+ */
+const ACCESS_MASK WinMasks[WIN_MASKS_TOTAL] =
+{
+  /* WIN_READ */
+  FILE_READ_DATA,
+  /* WIN_WRITE */
+  FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_APPEND_DATA | FILE_WRITE_EA |
+  FILE_DELETE_CHILD,
+  /* WIN_EXECUTE */
+  FILE_EXECUTE,
+  /* WIN_OWNER_SE */
+  DELETE | WRITE_DAC | WRITE_OWNER | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES, 
+  /* WIN_ALL */
+  READ_CONTROL |  FILE_READ_EA | FILE_READ_ATTRIBUTES | SYNCHRONIZE,
+};
+
+//----------------------------------------------------------------------------
+// Function: GetFileInformationByName
+//
+// Description:
+//  To retrieve the by handle file information given the file name
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  error code: otherwise
+//
+// Notes:
+//  If followLink parameter is set to TRUE, we will follow the symbolic link
+//  or junction point to get the target file information. Otherwise, the
+//  information for the symbolic link or junction point is retrieved.
+//
+DWORD GetFileInformationByName(
+  __in LPCWSTR pathName,
+  __in BOOL followLink,
+  __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation)
+{
+  HANDLE fileHandle = INVALID_HANDLE_VALUE;
+  BOOL isSymlink = FALSE;
+  BOOL isJunction = FALSE;
+  DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  assert(lpFileInformation != NULL);
+
+  if (!followLink)
+  {
+    if ((dwErrorCode = SymbolicLinkCheck(pathName, &isSymlink)) != ERROR_SUCCESS)
+      return dwErrorCode;
+    if ((dwErrorCode = JunctionPointCheck(pathName, &isJunction)) != ERROR_SUCCESS)
+      return dwErrorCode;
+    if (isSymlink || isJunction)
+      dwFlagsAndAttributes |= FILE_FLAG_OPEN_REPARSE_POINT;
+  }
+
+  fileHandle = CreateFileW(
+    pathName,
+    FILE_READ_ATTRIBUTES,
+    FILE_SHARE_READ,
+    NULL,
+    OPEN_EXISTING,
+    dwFlagsAndAttributes,
+    NULL);
+  if (fileHandle == INVALID_HANDLE_VALUE)
+  {
+    dwErrorCode = GetLastError();
+    return dwErrorCode;
+  }
+
+  if (!GetFileInformationByHandle(fileHandle, lpFileInformation))
+  {
+    dwErrorCode = GetLastError();
+    CloseHandle(fileHandle);
+    return dwErrorCode;
+  }
+
+  CloseHandle(fileHandle);
+
+  return dwErrorCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsLongWindowsPath
+//
+// Description:
+//  Checks if the path is longer than MAX_PATH in which case it needs to be
+//  prepended with \\?\ for Windows OS to understand it.
+//
+// Returns:
+//  TRUE long path
+//  FALSE otherwise
+static BOOL IsLongWindowsPath(__in PCWSTR path)
+{
+  return (wcslen(path) + 1) > MAX_PATH;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsPrefixedAlready
+//
+// Description:
+//  Checks if the given path is already prepended with \\?\.
+//
+// Returns:
+//  TRUE if yes
+//  FALSE otherwise
+static BOOL IsPrefixedAlready(__in PCWSTR path)
+{
+  static const PCWSTR LongPathPrefix = L"\\\\?\\";
+  size_t Prefixlen = wcslen(LongPathPrefix);
+  size_t i = 0;
+
+  if (path == NULL || wcslen(path) < Prefixlen)
+  {
+    return FALSE;
+  }
+
+  for (i = 0; i < Prefixlen; ++i)
+  {
+    if (path[i] != LongPathPrefix[i])
+    {
+      return FALSE;
+    }
+  }
+
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: ConvertToLongPath
+//
+// Description:
+//  Prepends the path with the \\?\ prefix if the path is longer than MAX_PATH.
+//  On success, newPath should be freed with LocalFree(). Given that relative
+//  paths cannot be longer than MAX_PATH, we will never prepend the prefix
+//  to relative paths.
+//
+// Returns:
+//  ERROR_SUCCESS on success
+//  error code on failure
+DWORD ConvertToLongPath(__in PCWSTR path, __deref_out PWSTR *newPath)
+{
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  static const PCWSTR LongPathPrefix = L"\\\\?\\";
+  BOOL bAppendPrefix = IsLongWindowsPath(path) && !IsPrefixedAlready(path);
+  HRESULT hr = S_OK;
+
+  size_t newPathLen = wcslen(path) + (bAppendPrefix ? wcslen(LongPathPrefix) : 0);
+
+  // Allocate the buffer for the output path (+1 for terminating NULL char)
+  //
+  PWSTR newPathValue = (PWSTR)LocalAlloc(LPTR, (newPathLen + 1) * sizeof(WCHAR));
+  if (newPathValue == NULL)
+  {
+    dwErrorCode = GetLastError();
+    goto ConvertToLongPathExit;
+  }
+
+  if (bAppendPrefix)
+  {
+    // Append the prefix to the path
+    //
+    hr = StringCchPrintfW(newPathValue, newPathLen + 1, L"%s%s",
+      LongPathPrefix, path);
+    if (FAILED(hr))
+    {
+      dwErrorCode = HRESULT_CODE(hr);
+      goto ConvertToLongPathExit;
+    }
+  }
+  else
+  {
+    // Just copy the original value into the output path. In this scenario
+    // we are doing extra buffer copy. We decided to trade code simplicity
+    // on the call site for small performance impact (extra allocation and
+    // buffer copy). As paths are short, the impact is generally small.
+    //
+    hr = StringCchPrintfW(newPathValue, newPathLen + 1, L"%s", path);
+    if (FAILED(hr))
+    {
+      dwErrorCode = HRESULT_CODE(hr);
+      goto ConvertToLongPathExit;
+    }
+  }
+
+  *newPath = newPathValue;
+
+ConvertToLongPathExit:
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    LocalFree(newPathValue);
+    *newPath = NULL;
+  }
+
+  return dwErrorCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsDirFileInfo
+//
+// Description:
+//	Test if the given file information is a directory
+//
+// Returns:
+//	TRUE if it is a directory
+//  FALSE otherwise
+//
+// Notes:
+//
+BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation)
+{
+  if ((fileInformation->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+    == FILE_ATTRIBUTE_DIRECTORY)
+    return TRUE;
+  return FALSE;
+}
+
+//----------------------------------------------------------------------------
+// Function: CheckFileAttributes
+//
+// Description:
+//	Check if the given file has all the given attribute(s)
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+static DWORD FileAttributesCheck(
+  __in LPCWSTR path, __in DWORD attr, __out PBOOL res)
+{
+  DWORD attrs = INVALID_FILE_ATTRIBUTES;
+  *res = FALSE;
+  if ((attrs = GetFileAttributes(path)) != INVALID_FILE_ATTRIBUTES)
+    *res = ((attrs & attr) == attr);
+  else
+    return GetLastError();
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsDirectory
+//
+// Description:
+//	Check if the given file is a directory
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+DWORD DirectoryCheck(__in LPCWSTR pathName, __out PBOOL res)
+{
+  return FileAttributesCheck(pathName, FILE_ATTRIBUTE_DIRECTORY, res);
+}
+
+//----------------------------------------------------------------------------
+// Function: IsReparsePoint
+//
+// Description:
+//	Check if the given file is a reparse point
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+static DWORD ReparsePointCheck(__in LPCWSTR pathName, __out PBOOL res)
+{
+  return FileAttributesCheck(pathName, FILE_ATTRIBUTE_REPARSE_POINT, res);
+}
+
+//----------------------------------------------------------------------------
+// Function: CheckReparseTag
+//
+// Description:
+//	Check if the given file is a reparse point of the given tag.
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+static DWORD ReparseTagCheck(__in LPCWSTR path, __in DWORD tag, __out PBOOL res)
+{
+  BOOL isReparsePoint = FALSE;
+  HANDLE hFind = INVALID_HANDLE_VALUE;
+  WIN32_FIND_DATA findData;
+  DWORD dwRtnCode;
+
+  if ((dwRtnCode = ReparsePointCheck(path, &isReparsePoint)) != ERROR_SUCCESS)
+    return dwRtnCode;
+
+  if (!isReparsePoint)
+  {
+    *res = FALSE;
+  }
+  else
+  {
+    if ((hFind = FindFirstFile(path, &findData)) == INVALID_HANDLE_VALUE)
+    {
+      return GetLastError();
+    }
+    else
+    {
+      *res = (findData.dwReserved0 == tag);
+      FindClose(hFind);
+    }
+  }
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsSymbolicLink
+//
+// Description:
+//	Check if the given file is a symbolic link.
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out PBOOL res)
+{
+  return ReparseTagCheck(pathName, IO_REPARSE_TAG_SYMLINK, res);
+}
+
+//----------------------------------------------------------------------------
+// Function: IsJunctionPoint
+//
+// Description:
+//	Check if the given file is a junction point.
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  error code otherwise
+//
+// Notes:
+//
+DWORD JunctionPointCheck(__in LPCWSTR pathName, __out PBOOL res)
+{
+  return ReparseTagCheck(pathName, IO_REPARSE_TAG_MOUNT_POINT, res);
+}
+
+//----------------------------------------------------------------------------
+// Function: GetSidFromAcctNameW
+//
+// Description:
+//	To retrieve the SID for a user account
+//
+// Returns:
+//	ERROR_SUCCESS: on success
+//  Other error code: otherwise
+//
+// Notes:
+//	Caller needs to destroy the memory of Sid by calling LocalFree()
+//
+DWORD GetSidFromAcctNameW(LPCWSTR acctName, PSID *ppSid)
+{
+  DWORD dwSidSize = 0;
+  DWORD cchDomainName = 0;
+  DWORD dwDomainNameSize = 0;
+  LPWSTR domainName = NULL;
+  SID_NAME_USE eSidType;
+
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  // Validate the input parameters.
+  //
+  assert (acctName != NULL && ppSid != NULL);
+
+  // Empty name is invalid. However, LookupAccountName() function will return a
+  // false Sid, i.e. Sid for 'BUILDIN', for an empty name instead failing. We
+  // report the error before calling LookupAccountName() function for this
+  // special case. The error code returned here is the same as the last error
+  // code set by LookupAccountName() function for an invalid name.
+  //
+  if (wcslen(acctName) == 0)
+    return ERROR_NONE_MAPPED;
+
+  // First pass to retrieve the buffer size.
+  //
+  LookupAccountName(
+    NULL, // Computer name. NULL for the local computer
+    acctName,
+    NULL, // pSid. NULL to retrieve buffer size
+    &dwSidSize,
+    NULL, // Domain Name. NULL to retrieve buffer size 
+    &cchDomainName,
+    &eSidType);
+
+  if((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER)
+  {
+    return dwErrorCode;
+  }
+  else
+  {
+    // Reallocate memory for the buffers.
+    //
+    *ppSid = (PSID)LocalAlloc(LPTR, dwSidSize);
+    if (*ppSid == NULL)
+    {
+      return GetLastError();
+    }
+    dwDomainNameSize = (cchDomainName + 1) * sizeof(wchar_t);
+    domainName = (LPWSTR)LocalAlloc(LPTR, dwDomainNameSize);
+    if (domainName == NULL)
+    {
+      return GetLastError();
+    }
+
+    // Second pass to retrieve the SID and domain name.
+    //
+    if (!LookupAccountNameW(
+      NULL, // Computer name. NULL for the local computer
+      acctName,
+      *ppSid,
+      &dwSidSize,
+      domainName, 
+      &cchDomainName,
+      &eSidType))
+    {
+      LocalFree(domainName);
+      return GetLastError();
+    }
+
+    assert(IsValidSid(*ppSid));
+  }
+
+  LocalFree(domainName);
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetUnixAccessMask
+//
+// Description:
+//	Compute the 3 bit Unix mask for the owner, group, or, others
+//
+// Returns:
+//	The 3 bit Unix mask in INT
+//
+// Notes:
+//
+static INT GetUnixAccessMask(ACCESS_MASK Mask)
+{
+  static const INT exe   = 0x0001;
+  static const INT write = 0x0002;
+  static const INT read  = 0x0004;
+  INT mask  = 0;
+
+  if ((Mask & WinMasks[WIN_READ]) == WinMasks[WIN_READ])
+    mask |= read;
+  if ((Mask & WinMasks[WIN_WRITE]) == WinMasks[WIN_WRITE])
+    mask |= write;
+  if ((Mask & WinMasks[WIN_EXECUTE]) == WinMasks[WIN_EXECUTE])
+    mask |= exe;
+  return mask;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetAccess
+//
+// Description:
+//	Get Windows acces mask by AuthZ methods
+//
+// Returns:
+//	ERROR_SUCCESS: on success
+//
+// Notes:
+//
+static DWORD GetAccess(AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClient,
+  PSECURITY_DESCRIPTOR psd, PACCESS_MASK pAccessRights)
+{
+  AUTHZ_ACCESS_REQUEST AccessRequest = {0};
+  AUTHZ_ACCESS_REPLY AccessReply = {0};
+  BYTE Buffer[1024];
+
+  assert (pAccessRights != NULL);
+
+  //  Do AccessCheck
+  AccessRequest.DesiredAccess = MAXIMUM_ALLOWED;
+  AccessRequest.PrincipalSelfSid = NULL;
+  AccessRequest.ObjectTypeList = NULL;
+  AccessRequest.ObjectTypeListLength = 0;
+  AccessRequest.OptionalArguments = NULL; 
+
+  RtlZeroMemory(Buffer, sizeof(Buffer));
+  AccessReply.ResultListLength = 1;
+  AccessReply.GrantedAccessMask = (PACCESS_MASK) (Buffer);
+  AccessReply.Error = (PDWORD) (Buffer + sizeof(ACCESS_MASK));
+
+  if (!AuthzAccessCheck(0,
+    hAuthzClient,
+    &AccessRequest,
+    NULL,
+    psd,
+    NULL,
+    0,
+    &AccessReply,
+    NULL))
+  {
+    return GetLastError();
+  }
+  *pAccessRights = (*(PACCESS_MASK)(AccessReply.GrantedAccessMask));
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetEffectiveRightsForSid
+//
+// Description:
+//	Get Windows acces mask by AuthZ methods
+//
+// Returns:
+//	ERROR_SUCCESS: on success
+//
+// Notes:
+//   We run into problems for local user accounts when using the method
+//   GetEffectiveRightsFromAcl(). We resort to using AuthZ methods as
+//   an alternative way suggested on MSDN:
+// http://msdn.microsoft.com/en-us/library/windows/desktop/aa446637.aspx
+//
+static DWORD GetEffectiveRightsForSid(PSECURITY_DESCRIPTOR psd,
+  PSID pSid,
+  PACCESS_MASK pAccessRights)
+{
+  AUTHZ_RESOURCE_MANAGER_HANDLE hManager;
+  LUID unusedId = { 0 };
+  AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClientContext = NULL;
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  DWORD ret = ERROR_SUCCESS;
+
+  assert (pAccessRights != NULL);
+
+  if (!AuthzInitializeResourceManager(AUTHZ_RM_FLAG_NO_AUDIT,
+    NULL, NULL, NULL, NULL, &hManager))
+  {
+    return GetLastError();
+  }
+
+  if(!AuthzInitializeContextFromSid(AUTHZ_SKIP_TOKEN_GROUPS,
+    pSid, hManager, NULL, unusedId, NULL, &hAuthzClientContext))
+  {
+    ret = GetLastError();
+    goto GetEffectiveRightsForSidEnd;
+  }
+
+  if ((dwRtnCode = GetAccess(hAuthzClientContext, psd, pAccessRights))
+    != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto GetEffectiveRightsForSidEnd;
+  }
+  if (!AuthzFreeContext(hAuthzClientContext))
+  {
+    ret = GetLastError();
+    goto GetEffectiveRightsForSidEnd;
+  }
+
+GetEffectiveRightsForSidEnd:
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: FindFileOwnerAndPermission
+//
+// Description:
+//	Find the owner, primary group and permissions of a file object
+//
+// Returns:
+//	ERROR_SUCCESS: on success
+//  Error code otherwise
+//
+// Notes:
+//  - Caller needs to destroy the memeory of owner and group names by calling
+//    LocalFree() function.
+//
+//  - If the user or group name does not exist, the user or group SID will be
+//    returned as the name.
+//
+DWORD FindFileOwnerAndPermission(
+  __in LPCWSTR pathName,
+  __out_opt LPWSTR *pOwnerName,
+  __out_opt LPWSTR *pGroupName,
+  __out_opt PINT pMask)
+{
+  DWORD dwRtnCode = 0;
+
+  PSECURITY_DESCRIPTOR pSd = NULL;
+
+  PSID psidOwner = NULL;
+  PSID psidGroup = NULL;
+  PSID psidEveryone = NULL;
+  DWORD cbSid = SECURITY_MAX_SID_SIZE;
+  PACL pDacl = NULL;
+
+  ACCESS_MASK ownerAccessRights = 0;
+  ACCESS_MASK groupAccessRights = 0;
+  ACCESS_MASK worldAccessRights = 0;
+
+  DWORD ret = ERROR_SUCCESS;
+
+  // Do nothing if the caller request nothing
+  //
+  if (pOwnerName == NULL && pGroupName == NULL && pMask == NULL)
+  {
+    return ret;
+  }
+
+  dwRtnCode = GetNamedSecurityInfo(pathName, SE_FILE_OBJECT,
+    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION |
+    DACL_SECURITY_INFORMATION,
+    &psidOwner, &psidGroup, &pDacl, NULL, &pSd);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto FindFileOwnerAndPermissionEnd;
+  }
+
+  if (pOwnerName != NULL)
+  {
+    dwRtnCode = GetAccntNameFromSid(psidOwner, pOwnerName);
+    if (dwRtnCode == ERROR_NONE_MAPPED)
+    {
+      if (!ConvertSidToStringSid(psidOwner, pOwnerName))
+      {
+        ret = GetLastError();
+        goto FindFileOwnerAndPermissionEnd;
+      }
+    }
+    else if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ret = dwRtnCode;
+      goto FindFileOwnerAndPermissionEnd;
+    }
+  }
+
+  if (pGroupName != NULL)
+  {
+    dwRtnCode = GetAccntNameFromSid(psidGroup, pGroupName);
+    if (dwRtnCode == ERROR_NONE_MAPPED)
+    {
+      if (!ConvertSidToStringSid(psidGroup, pGroupName))
+      {
+        ret = GetLastError();
+        goto FindFileOwnerAndPermissionEnd;
+      }
+    }
+    else if (dwRtnCode != ERROR_SUCCESS)
+    {
+      ret = dwRtnCode;
+      goto FindFileOwnerAndPermissionEnd;
+    }
+  }
+
+  if (pMask == NULL) goto FindFileOwnerAndPermissionEnd;
+
+  if ((dwRtnCode = GetEffectiveRightsForSid(pSd,
+    psidOwner, &ownerAccessRights)) != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto FindFileOwnerAndPermissionEnd;
+  }
+
+  if ((dwRtnCode = GetEffectiveRightsForSid(pSd,
+    psidGroup, &groupAccessRights)) != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto FindFileOwnerAndPermissionEnd;
+  }
+
+  if ((psidEveryone = LocalAlloc(LPTR, cbSid)) == NULL)
+  {
+    ret = GetLastError();
+    goto FindFileOwnerAndPermissionEnd;
+  }
+  if (!CreateWellKnownSid(WinWorldSid, NULL, psidEveryone, &cbSid))
+  {
+    ret = GetLastError();
+    goto FindFileOwnerAndPermissionEnd;
+  }
+  if ((dwRtnCode = GetEffectiveRightsForSid(pSd,
+    psidEveryone, &worldAccessRights)) != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto FindFileOwnerAndPermissionEnd;
+  }
+
+  *pMask |= GetUnixAccessMask(ownerAccessRights) << 6;
+  *pMask |= GetUnixAccessMask(groupAccessRights) << 3;
+  *pMask |= GetUnixAccessMask(worldAccessRights);
+
+FindFileOwnerAndPermissionEnd:
+  LocalFree(psidEveryone);
+  LocalFree(pSd);
+
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetWindowsAccessMask
+//
+// Description:
+//  Get the Windows AccessMask for user, group and everyone based on the Unix
+//  permission mask
+//
+// Returns:
+//  none
+//
+// Notes:
+//  none
+//
+static void GetWindowsAccessMask(INT unixMask,
+  ACCESS_MASK *userAllow,
+  ACCESS_MASK *userDeny,
+  ACCESS_MASK *groupAllow,
+  ACCESS_MASK *groupDeny,
+  ACCESS_MASK *otherAllow)
+{
+  assert (userAllow != NULL && userDeny != NULL &&
+    groupAllow != NULL && groupDeny != NULL &&
+    otherAllow != NULL);
+
+  *userAllow = WinMasks[WIN_ALL] | WinMasks[WIN_OWNER_SE];
+  if ((unixMask & UX_U_READ) == UX_U_READ)
+    *userAllow |= WinMasks[WIN_READ];
+
+  if ((unixMask & UX_U_WRITE) == UX_U_WRITE)
+    *userAllow |= WinMasks[WIN_WRITE];
+
+  if ((unixMask & UX_U_EXECUTE) == UX_U_EXECUTE)
+    *userAllow |= WinMasks[WIN_EXECUTE];
+
+  *userDeny = 0;
+  if ((unixMask & UX_U_READ) != UX_U_READ &&
+    ((unixMask & UX_G_READ) == UX_G_READ ||
+    (unixMask & UX_O_READ) == UX_O_READ))
+    *userDeny |= WinMasks[WIN_READ];
+
+  if ((unixMask & UX_U_WRITE) != UX_U_WRITE &&
+    ((unixMask & UX_G_WRITE) == UX_G_WRITE ||
+    (unixMask & UX_O_WRITE) == UX_O_WRITE))
+    *userDeny |= WinMasks[WIN_WRITE];
+
+  if ((unixMask & UX_U_EXECUTE) != UX_U_EXECUTE &&
+    ((unixMask & UX_G_EXECUTE) == UX_G_EXECUTE ||
+    (unixMask & UX_O_EXECUTE) == UX_O_EXECUTE))
+    *userDeny |= WinMasks[WIN_EXECUTE];
+
+  *groupAllow = WinMasks[WIN_ALL];
+  if ((unixMask & UX_G_READ) == UX_G_READ)
+    *groupAllow |= FILE_GENERIC_READ;
+
+  if ((unixMask & UX_G_WRITE) == UX_G_WRITE)
+    *groupAllow |= WinMasks[WIN_WRITE];
+
+  if ((unixMask & UX_G_EXECUTE) == UX_G_EXECUTE)
+    *groupAllow |= WinMasks[WIN_EXECUTE];
+
+  *groupDeny = 0;
+  if ((unixMask & UX_G_READ) != UX_G_READ &&
+    (unixMask & UX_O_READ) == UX_O_READ)
+    *groupDeny |= WinMasks[WIN_READ];
+
+  if ((unixMask & UX_G_WRITE) != UX_G_WRITE &&
+    (unixMask & UX_O_WRITE) == UX_O_WRITE)
+    *groupDeny |= WinMasks[WIN_WRITE];
+
+  if ((unixMask & UX_G_EXECUTE) != UX_G_EXECUTE &&
+    (unixMask & UX_O_EXECUTE) == UX_O_EXECUTE)
+    *groupDeny |= WinMasks[WIN_EXECUTE];
+
+  *otherAllow = WinMasks[WIN_ALL];
+  if ((unixMask & UX_O_READ) == UX_O_READ)
+    *otherAllow |= WinMasks[WIN_READ];
+
+  if ((unixMask & UX_O_WRITE) == UX_O_WRITE)
+    *otherAllow |= WinMasks[WIN_WRITE];
+
+  if ((unixMask & UX_O_EXECUTE) == UX_O_EXECUTE)
+    *otherAllow |= WinMasks[WIN_EXECUTE];
+}
+
+//----------------------------------------------------------------------------
+// Function: GetWindowsDACLs
+//
+// Description:
+//  Get the Windows DACs based the Unix access mask
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+// Notes:
+//  - Administrators and SYSTEM are always given full permission to the file,
+//    unless Administrators or SYSTEM itself is the file owner and the user
+//    explictly set the permission to something else. For example, file 'foo'
+//    belongs to Administrators, 'chmod 000' on the file will not directly
+//    assign Administrators full permission on the file.
+//  - Only full permission for Administrators and SYSTEM are inheritable.
+//  - CREATOR OWNER is always given full permission and the permission is
+//    inheritable, more specifically OBJECT_INHERIT_ACE, CONTAINER_INHERIT_ACE
+//    flags are set. The reason is to give the creator of child file full
+//    permission, i.e., the child file will have permission mode 700 for
+//    a user other than Administrator or SYSTEM.
+//
+static DWORD GetWindowsDACLs(__in INT unixMask,
+  __in PSID pOwnerSid, __in PSID pGroupSid, __out PACL *ppNewDACL)
+{
+  DWORD winUserAccessDenyMask;
+  DWORD winUserAccessAllowMask;
+  DWORD winGroupAccessDenyMask;
+  DWORD winGroupAccessAllowMask;
+  DWORD winOtherAccessAllowMask;
+
+  PSID pEveryoneSid = NULL;
+  DWORD cbEveryoneSidSize = SECURITY_MAX_SID_SIZE;
+
+  PSID pSystemSid = NULL;
+  DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE;
+  BOOL bAddSystemAcls = FALSE;
+
+  PSID pAdministratorsSid = NULL;
+  DWORD cbAdministratorsSidSize = SECURITY_MAX_SID_SIZE;
+  BOOL bAddAdministratorsAcls = FALSE;
+
+  PSID pCreatorOwnerSid = NULL;
+  DWORD cbCreatorOwnerSidSize = SECURITY_MAX_SID_SIZE;
+
+  PACL pNewDACL = NULL;
+  DWORD dwNewAclSize = 0;
+
+  DWORD ret = ERROR_SUCCESS;
+
+  GetWindowsAccessMask(unixMask,
+    &winUserAccessAllowMask, &winUserAccessDenyMask,
+    &winGroupAccessAllowMask, &winGroupAccessDenyMask,
+    &winOtherAccessAllowMask);
+
+  // Create a well-known SID for the Everyone group
+  //
+  if ((pEveryoneSid = LocalAlloc(LPTR, cbEveryoneSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinWorldSid, NULL, pEveryoneSid, &cbEveryoneSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  // Create a well-known SID for the Administrators group
+  //
+  if ((pAdministratorsSid = LocalAlloc(LPTR, cbAdministratorsSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL,
+    pAdministratorsSid, &cbAdministratorsSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!EqualSid(pAdministratorsSid, pOwnerSid)
+    && !EqualSid(pAdministratorsSid, pGroupSid))
+    bAddAdministratorsAcls = TRUE;
+
+  // Create a well-known SID for the SYSTEM
+  //
+  if ((pSystemSid = LocalAlloc(LPTR, cbSystemSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinLocalSystemSid, NULL,
+    pSystemSid, &cbSystemSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!EqualSid(pSystemSid, pOwnerSid)
+    && !EqualSid(pSystemSid, pGroupSid))
+    bAddSystemAcls = TRUE;
+
+  // Create a well-known SID for the Creator Owner
+  //
+  if ((pCreatorOwnerSid = LocalAlloc(LPTR, cbCreatorOwnerSidSize)) == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!CreateWellKnownSid(WinCreatorOwnerSid, NULL,
+    pCreatorOwnerSid, &cbCreatorOwnerSidSize))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  // Create the new DACL
+  //
+  dwNewAclSize = sizeof(ACL);
+  dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+    GetLengthSid(pOwnerSid) - sizeof(DWORD);
+  if (winUserAccessDenyMask)
+    dwNewAclSize += sizeof(ACCESS_DENIED_ACE) +
+    GetLengthSid(pOwnerSid) - sizeof(DWORD);
+  dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+    GetLengthSid(pGroupSid) - sizeof(DWORD);
+  if (winGroupAccessDenyMask)
+    dwNewAclSize += sizeof(ACCESS_DENIED_ACE) +
+    GetLengthSid(pGroupSid) - sizeof(DWORD);
+  dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+    GetLengthSid(pEveryoneSid) - sizeof(DWORD);
+
+  if (bAddSystemAcls)
+  {
+    dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+      cbSystemSidSize - sizeof(DWORD);
+  }
+
+  if (bAddAdministratorsAcls)
+  {
+    dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+      cbAdministratorsSidSize - sizeof(DWORD);
+  }
+
+  dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) +
+    cbCreatorOwnerSidSize - sizeof(DWORD);
+
+  pNewDACL = (PACL)LocalAlloc(LPTR, dwNewAclSize);
+  if (pNewDACL == NULL)
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!InitializeAcl(pNewDACL, dwNewAclSize, ACL_REVISION))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pCreatorOwnerSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (bAddSystemAcls &&
+    !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pSystemSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (bAddAdministratorsAcls &&
+    !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
+    GENERIC_ALL, pAdministratorsSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  if (winUserAccessDenyMask &&
+    !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
+    winUserAccessDenyMask, pOwnerSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
+    winUserAccessAllowMask, pOwnerSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (winGroupAccessDenyMask &&
+    !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
+    winGroupAccessDenyMask, pGroupSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
+    winGroupAccessAllowMask, pGroupSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+  if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
+    NO_PROPAGATE_INHERIT_ACE,
+    winOtherAccessAllowMask, pEveryoneSid))
+  {
+    ret = GetLastError();
+    goto GetWindowsDACLsEnd;
+  }
+
+  *ppNewDACL = pNewDACL;
+
+GetWindowsDACLsEnd:
+  LocalFree(pEveryoneSid);
+  LocalFree(pAdministratorsSid);
+  LocalFree(pSystemSid);
+  LocalFree(pCreatorOwnerSid);
+  if (ret != ERROR_SUCCESS) LocalFree(pNewDACL);
+  
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileModeByMask
+//
+// Description:
+//  Change a file or direcotry at the path to Unix mode
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+// Notes:
+//  This function is long path safe, i.e. the path will be converted to long
+//  path format if not already converted. So the caller does not need to do
+//  the converstion before calling the method.
+//
+DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode)
+{
+  LPWSTR longPathName = NULL;
+  PACL pNewDACL = NULL;
+  PSID pOwnerSid = NULL;
+  PSID pGroupSid = NULL;
+  PSECURITY_DESCRIPTOR pSD = NULL;
+
+  SECURITY_DESCRIPTOR_CONTROL control;
+  DWORD revision = 0;
+
+  PSECURITY_DESCRIPTOR pAbsSD = NULL;
+  PACL pAbsDacl = NULL;
+  PACL pAbsSacl = NULL;
+  PSID pAbsOwner = NULL;
+  PSID pAbsGroup = NULL;
+
+  DWORD dwRtnCode = 0;
+  DWORD dwErrorCode = 0;
+
+  DWORD ret = ERROR_SUCCESS;
+
+  dwRtnCode = ConvertToLongPath(path, &longPathName);
+  if (dwRtnCode != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto ChangeFileModeByMaskEnd;
+  }
+
+  // Get owner and group Sids
+  //
+  dwRtnCode = GetNamedSecurityInfoW(
+    longPathName,
+    SE_FILE_OBJECT, 
+    OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION,
+    &pOwnerSid,
+    &pGroupSid,
+    NULL,
+    NULL,
+    &pSD);
+  if (ERROR_SUCCESS != dwRtnCode)
+  {
+    ret = dwRtnCode;
+    goto ChangeFileModeByMaskEnd; 
+  }
+
+  // SetSecurityDescriptorDacl function used below only accepts security
+  // descriptor in absolute format, meaning that its members must be pointers to
+  // other structures, rather than offsets to contiguous data.
+  // To determine whether a security descriptor is self-relative or absolute,
+  // call the GetSecurityDescriptorControl function and check the
+  // SE_SELF_RELATIVE flag of the SECURITY_DESCRIPTOR_CONTROL parameter.
+  //
+  if (!GetSecurityDescriptorControl(pSD, &control, &revision))
+  {
+    ret = GetLastError();
+    goto ChangeFileModeByMaskEnd;
+  }
+
+  // If the security descriptor is self-relative, we use MakeAbsoluteSD function
+  // to convert it to absolute format.
+  //
+  if ((control & SE_SELF_RELATIVE) == SE_SELF_RELATIVE)
+  {
+    DWORD absSDSize = 0;
+    DWORD daclSize = 0;
+    DWORD saclSize = 0;
+    DWORD ownerSize = 0;
+    DWORD primaryGroupSize = 0;
+    MakeAbsoluteSD(pSD, NULL, &absSDSize, NULL, &daclSize, NULL,
+      &saclSize, NULL, &ownerSize, NULL, &primaryGroupSize);
+    if ((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER)
+    {
+      ret = dwErrorCode;
+      goto ChangeFileModeByMaskEnd;
+    }
+
+    if ((pAbsSD = (PSECURITY_DESCRIPTOR) LocalAlloc(LPTR, absSDSize)) == NULL)
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+    if ((pAbsDacl = (PACL) LocalAlloc(LPTR, daclSize)) == NULL)
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+    if ((pAbsSacl = (PACL) LocalAlloc(LPTR, saclSize)) == NULL)
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+    if ((pAbsOwner = (PSID) LocalAlloc(LPTR, ownerSize)) == NULL)
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+    if ((pAbsGroup = (PSID) LocalAlloc(LPTR, primaryGroupSize)) == NULL)
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+
+    if (!MakeAbsoluteSD(pSD, pAbsSD, &absSDSize, pAbsDacl, &daclSize, pAbsSacl,
+      &saclSize, pAbsOwner, &ownerSize, pAbsGroup, &primaryGroupSize))
+    {
+      ret = GetLastError();
+      goto ChangeFileModeByMaskEnd;
+    }
+  }
+
+  // Get Windows DACLs based on Unix access mask
+  //
+  if ((dwRtnCode = GetWindowsDACLs(mode, pOwnerSid, pGroupSid, &pNewDACL))
+    != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto ChangeFileModeByMaskEnd;
+  }
+
+  // Set the DACL information in the security descriptor; if a DACL is already
+  // present in the security descriptor, the DACL is replaced. The security
+  // descriptor is then used to set the security of a file or directory.
+  //
+  if (!SetSecurityDescriptorDacl(pAbsSD, TRUE, pNewDACL, FALSE))
+  {
+    ret = GetLastError();
+    goto ChangeFileModeByMaskEnd;
+  }
+
+  // MSDN states "This function is obsolete. Use the SetNamedSecurityInfo
+  // function instead." However we have the following problem when using
+  // SetNamedSecurityInfo:
+  //  - When PROTECTED_DACL_SECURITY_INFORMATION is not passed in as part of
+  //    security information, the object will include inheritable permissions
+  //    from its parent.
+  //  - When PROTECTED_DACL_SECURITY_INFORMATION is passsed in to set
+  //    permissions on a directory, the child object of the directory will lose
+  //    inheritable permissions from their parent (the current directory).
+  // By using SetFileSecurity, we have the nice property that the new
+  // permissions of the object does not include the inheritable permissions from
+  // its parent, and the child objects will not lose their inherited permissions
+  // from the current object.
+  //
+  if (!SetFileSecurity(longPathName, DACL_SECURITY_INFORMATION, pAbsSD))
+  {
+    ret = GetLastError();
+    goto ChangeFileModeByMaskEnd;
+  }
+
+ChangeFileModeByMaskEnd:
+  LocalFree(longPathName);
+  LocalFree(pSD);
+  LocalFree(pNewDACL);
+  LocalFree(pAbsDacl);
+  LocalFree(pAbsSacl);
+  LocalFree(pAbsOwner);
+  LocalFree(pAbsGroup);
+  LocalFree(pAbsSD);
+
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetAccntNameFromSid
+//
+// Description:
+//	To retrieve an account name given the SID
+//
+// Returns:
+//	ERROR_SUCCESS: on success
+//  Other error code: otherwise
+//
+// Notes:
+//	Caller needs to destroy the memory of account name by calling LocalFree()
+//
+DWORD GetAccntNameFromSid(PSID pSid, LPWSTR *ppAcctName)
+{
+  LPWSTR lpName = NULL;
+  DWORD cchName = 0;
+  LPWSTR lpDomainName = NULL;
+  DWORD cchDomainName = 0;
+  SID_NAME_USE eUse = SidTypeUnknown;
+  DWORD cchAcctName = 0;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  HRESULT hr = S_OK;
+
+  DWORD ret = ERROR_SUCCESS;
+
+  assert(ppAcctName != NULL);
+
+  // NOTE:
+  // MSDN says the length returned for the buffer size including the terminating
+  // null character. However we found it is not true during debuging.
+  //
+  LookupAccountSid(NULL, pSid, NULL, &cchName, NULL, &cchDomainName, &eUse);
+  if ((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER)
+    return dwErrorCode;
+  lpName = (LPWSTR) LocalAlloc(LPTR, (cchName + 1) * sizeof(WCHAR));
+  if (lpName == NULL)
+  {
+    ret = GetLastError();
+    goto GetAccntNameFromSidEnd;
+  }
+  lpDomainName = (LPWSTR) LocalAlloc(LPTR, (cchDomainName + 1) * sizeof(WCHAR));
+  if (lpDomainName == NULL)
+  {
+    ret = GetLastError();
+    goto GetAccntNameFromSidEnd;
+  }
+
+  if (!LookupAccountSid(NULL, pSid,
+    lpName, &cchName, lpDomainName, &cchDomainName, &eUse))
+  {
+    ret = GetLastError();
+    goto GetAccntNameFromSidEnd;
+  }
+
+  // Buffer size = name length + 1 for '\' + domain length + 1 for NULL
+  cchAcctName = cchName + cchDomainName + 2;
+  *ppAcctName = (LPWSTR) LocalAlloc(LPTR, cchAcctName * sizeof(WCHAR));
+  if (*ppAcctName == NULL)
+  {
+    ret = GetLastError();
+    goto GetAccntNameFromSidEnd;
+  }
+
+  hr = StringCchCopyW(*ppAcctName, cchAcctName, lpDomainName);
+  if (FAILED(hr))
+  {
+    ret = HRESULT_CODE(hr);
+    goto GetAccntNameFromSidEnd;
+  }
+
+  hr = StringCchCatW(*ppAcctName, cchAcctName, L"\\");
+  if (FAILED(hr))
+  {
+    ret = HRESULT_CODE(hr);
+    goto GetAccntNameFromSidEnd;
+  }
+
+  hr = StringCchCatW(*ppAcctName, cchAcctName, lpName);
+  if (FAILED(hr))
+  {
+    ret = HRESULT_CODE(hr);
+    goto GetAccntNameFromSidEnd;
+  }
+
+GetAccntNameFromSidEnd:
+  LocalFree(lpName);
+  LocalFree(lpDomainName);
+  if (ret != ERROR_SUCCESS)
+  {
+    LocalFree(*ppAcctName);
+    *ppAcctName = NULL;
+  }
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetLocalGroupsForUser
+//
+// Description:
+//	Get an array of groups for the given user.
+//
+// Returns:
+//	ERROR_SUCCESS on success
+//  Other error code on failure
+//
+// Notes:
+// - NetUserGetLocalGroups() function only accepts full user name in the format
+//   [domain name]\[username]. The user input to this function can be only the
+//   username. In this case, NetUserGetLocalGroups() will fail on the first try,
+//   and we will try to find full user name using LookupAccountNameW() method,
+//   and call NetUserGetLocalGroups() function again with full user name.
+//   However, it is not always possible to find full user name given only user
+//   name. For example, a computer named 'win1' joined domain 'redmond' can have
+//   two different users, 'win1\alex' and 'redmond\alex'. Given only 'alex', we
+//   cannot tell which one is correct.
+//
+// - Caller needs to destroy the memory of groups by using the
+//   NetApiBufferFree() function
+//
+DWORD GetLocalGroupsForUser(
+  __in LPCWSTR user,
+  __out LPLOCALGROUP_USERS_INFO_0 *groups,
+  __out LPDWORD entries)
+{
+  DWORD dwEntriesRead = 0;
+  DWORD dwTotalEntries = 0;
+  NET_API_STATUS nStatus = NERR_Success;
+
+  PSID pUserSid = NULL;
+  LPWSTR fullName = NULL;
+
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  DWORD ret = ERROR_SUCCESS;
+
+  *groups = NULL;
+  *entries = 0;
+
+  nStatus = NetUserGetLocalGroups(NULL,
+    user,
+    0,
+    0,
+    (LPBYTE *) groups,
+    MAX_PREFERRED_LENGTH,
+    &dwEntriesRead,
+    &dwTotalEntries);
+
+  if (nStatus == NERR_Success)
+  {
+    *entries = dwEntriesRead;
+    return ERROR_SUCCESS;
+  }
+  else if (nStatus != NERR_UserNotFound)
+  {
+    return nStatus;
+  }
+
+  if ((dwRtnCode = GetSidFromAcctNameW(user, &pUserSid)) != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto GetLocalGroupsForUserEnd;
+  }
+
+  if ((dwRtnCode = GetAccntNameFromSid(pUserSid, &fullName)) != ERROR_SUCCESS)
+  {
+    ret = dwRtnCode;
+    goto GetLocalGroupsForUserEnd;
+  }
+
+  nStatus = NetUserGetLocalGroups(NULL,
+    fullName,
+    0,
+    0,
+    (LPBYTE *) groups,
+    MAX_PREFERRED_LENGTH,
+    &dwEntriesRead,
+    &dwTotalEntries);
+  if (nStatus != NERR_Success)
+  {
+    // NERR_DCNotFound (2453) and NERR_UserNotFound (2221) are not published
+    // Windows System Error Code. All other error codes returned by
+    // NetUserGetLocalGroups() are valid System Error Codes according to MSDN.
+    ret = nStatus;
+    goto GetLocalGroupsForUserEnd;
+  }
+
+  *entries = dwEntriesRead;
+
+GetLocalGroupsForUserEnd:
+  LocalFree(pUserSid);
+  LocalFree(fullName);
+  return ret;
+}
+
+//----------------------------------------------------------------------------
+// Function: EnablePrivilege
+//
+// Description:
+//	Check if the process has the given privilege. If yes, enable the privilege
+//  to the process's access token.
+//
+// Returns:
+//	TRUE: on success
+//
+// Notes:
+//
+BOOL EnablePrivilege(__in LPCWSTR privilegeName)
+{
+  HANDLE hToken = INVALID_HANDLE_VALUE;
+  TOKEN_PRIVILEGES tp = { 0 };
+  DWORD dwErrCode = ERROR_SUCCESS;
+
+  if (!OpenProcessToken(GetCurrentProcess(),
+    TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken))
+  {
+    ReportErrorCode(L"OpenProcessToken", GetLastError());
+    return FALSE;
+  }
+
+  tp.PrivilegeCount = 1;
+  if (!LookupPrivilegeValueW(NULL,
+    privilegeName, &(tp.Privileges[0].Luid)))
+  {
+    ReportErrorCode(L"LookupPrivilegeValue", GetLastError());
+    CloseHandle(hToken);
+    return FALSE;
+  }
+  tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+
+  // As stated on MSDN, we need to use GetLastError() to check if
+  // AdjustTokenPrivileges() adjusted all of the specified privileges.
+  //
+  AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL);
+  dwErrCode = GetLastError();
+  CloseHandle(hToken);
+
+  return dwErrCode == ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: ReportErrorCode
+//
+// Description:
+//  Report an error. Use FormatMessage function to get the system error message.
+//
+// Returns:
+//  None
+//
+// Notes:
+//
+//
+void ReportErrorCode(LPCWSTR func, DWORD err)
+{
+  DWORD len = 0;
+  LPWSTR msg = NULL;
+
+  assert(func != NULL);
+
+  len = FormatMessageW(
+    FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    NULL, err,
+    MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+    (LPWSTR)&msg, 0, NULL);
+  if (len > 0)
+  {
+    fwprintf(stderr, L"%s error (%d): %s\n", func, err, msg);
+  }
+  else
+  {
+    fwprintf(stderr, L"%s error code: %d.\n", func, err);
+  }
+  if (msg != NULL) LocalFree(msg);
+}

hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj

@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{12131AA7-902E-4a6d-9CE3-043261D22A12}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>winutils</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IncludePath>include;$(IncludePath)</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <OutDir />
+    <IntDir>..\..\..\target\winutils\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <OutDir>..\..\..\target\bin\</OutDir>
+    <IntDir>..\..\..\target\winutils\$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="libwinutils.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="include/winutils.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

hadoop-common-project/hadoop-common/src/main/winutils/ls.c

@@ -0,0 +1,346 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#include "winutils.h"
+
+//----------------------------------------------------------------------------
+// Function: GetMaskString
+//
+// Description:
+//	Get the mask string that are used for output to the console.
+//
+// Returns:
+//	TRUE: on success
+//
+// Notes:
+//  The function only sets the existed permission in the mask string. If the
+//  permission does not exist, the corresponding character in mask string is not
+//  altered. The caller need to initilize the mask string to be all '-' to get
+//  the correct mask string.
+//
+static BOOL GetMaskString(INT accessMask, LPWSTR maskString)
+{
+  if(wcslen(maskString) != 10)
+    return FALSE;
+
+  if ((accessMask & UX_DIRECTORY) == UX_DIRECTORY)
+    maskString[0] = L'd';
+  else if ((accessMask & UX_SYMLINK) == UX_SYMLINK)
+    maskString[0] = L'l';
+
+  if ((accessMask & UX_U_READ) == UX_U_READ)
+    maskString[1] = L'r';
+  if ((accessMask & UX_U_WRITE) == UX_U_WRITE)
+    maskString[2] = L'w';
+  if ((accessMask & UX_U_EXECUTE) == UX_U_EXECUTE)
+    maskString[3] = L'x';
+
+  if ((accessMask & UX_G_READ) == UX_G_READ)
+    maskString[4] = L'r';
+  if ((accessMask & UX_G_WRITE) == UX_G_WRITE)
+    maskString[5] = L'w';
+  if ((accessMask & UX_G_EXECUTE) == UX_G_EXECUTE)
+    maskString[6] = L'x';
+
+  if ((accessMask & UX_O_READ) == UX_O_READ)
+    maskString[7] = L'r';
+  if ((accessMask & UX_O_WRITE) == UX_O_WRITE)
+    maskString[8] = L'w';
+  if ((accessMask & UX_O_EXECUTE) == UX_O_EXECUTE)
+    maskString[9] = L'x';
+
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: LsPrintLine
+//
+// Description:
+//	Print one line of 'ls' command given all the information needed
+//
+// Returns:
+//	None
+//
+// Notes:
+//  if useSeparator is false, separates the output tokens with a space
+//  character, otherwise, with a pipe character
+//
+static BOOL LsPrintLine(
+  const INT mask,
+  const DWORD hardlinkCount,
+  LPCWSTR ownerName,
+  LPCWSTR groupName,
+  const FILETIME *lpFileWritetime,
+  const LARGE_INTEGER fileSize,
+  LPCWSTR path,
+  BOOL useSeparator)
+{
+  // 'd' + 'rwx' for user, group, other
+  static const size_t ck_ullMaskLen = 1 + 3 * 3;
+
+  LPWSTR maskString = NULL;
+  SYSTEMTIME stFileWriteTime;
+  BOOL ret = FALSE;
+
+  maskString = (LPWSTR)LocalAlloc(LPTR, (ck_ullMaskLen+1)*sizeof(WCHAR));
+  if (maskString == NULL)
+  {
+    ReportErrorCode(L"LocalAlloc", GetLastError());
+    return FALSE;
+  }
+
+  // Build mask string from mask mode
+  if (FAILED(StringCchCopyW(maskString, (ck_ullMaskLen+1), L"----------")))
+  {
+    goto LsPrintLineEnd;
+  }
+
+  if (!GetMaskString(mask, maskString))
+  {
+    goto LsPrintLineEnd;
+  }
+
+  // Convert file time to system time
+  if (!FileTimeToSystemTime(lpFileWritetime, &stFileWriteTime))
+  {
+    goto LsPrintLineEnd;
+  }
+
+  if (useSeparator)
+  {
+    fwprintf(stdout, L"%10s|%d|%s|%s|%lld|%3s|%2d|%4d|%s\n",
+      maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart,
+      MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay,
+      stFileWriteTime.wYear, path);
+  }
+  else
+  {
+    fwprintf(stdout, L"%10s %d %s %s %lld %3s %2d %4d %s\n",
+      maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart,
+      MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay,
+      stFileWriteTime.wYear, path);
+  }
+
+  ret = TRUE;
+
+LsPrintLineEnd:
+  LocalFree(maskString);
+
+  return ret;
+}
+
+// List of command line options supported by "winutils ls"
+enum CmdLineOption
+{
+  CmdLineOptionFollowSymlink = 0x1,  // "-L"
+  CmdLineOptionSeparator = 0x2  // "-F"
+  // options should be powers of 2 (aka next is 0x4)
+};
+
+static wchar_t* CurrentDir = L".";
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//   Parses the command line
+//
+// Returns:
+//   TRUE on the valid command line, FALSE otherwise
+//
+BOOL ParseCommandLine(
+  int argc, wchar_t *argv[], wchar_t** path, int *optionsMask)
+{
+  int MaxOptions = 2; // Should be equal to the number of elems in CmdLineOption
+  int i = 0;
+
+  assert(optionsMask != NULL);
+  assert(argv != NULL);
+  assert(path != NULL);
+
+  *optionsMask = 0;
+
+  if (argc == 1)
+  {
+    // no path specified, assume "."
+    *path = CurrentDir;
+    return TRUE;
+  }
+
+  if (argc == 2)
+  {
+    // only path specified, no other options
+    *path = argv[1];
+    return TRUE;
+  }
+
+  if (argc > 2 + MaxOptions)
+  {
+    // too many parameters
+    return FALSE;
+  }
+
+  for (i = 1; i < argc - 1; ++i)
+  {
+    if (wcscmp(argv[i], L"-L") == 0)
+    {
+      // Check if this option was already specified
+      BOOL alreadySet = *optionsMask & CmdLineOptionFollowSymlink;
+      if (alreadySet)
+        return FALSE;
+
+      *optionsMask |= CmdLineOptionFollowSymlink;
+    }
+    else if (wcscmp(argv[i], L"-F") == 0)
+    {
+      // Check if this option was already specified
+      BOOL alreadySet = *optionsMask & CmdLineOptionSeparator;
+      if (alreadySet)
+        return FALSE;
+
+      *optionsMask |= CmdLineOptionSeparator;
+    }
+    else
+    {
+      return FALSE;
+    }
+  }
+
+  *path = argv[argc - 1];
+
+  return TRUE;
+}
+
+//----------------------------------------------------------------------------
+// Function: Ls
+//
+// Description:
+//	The main method for ls command
+//
+// Returns:
+//	0: on success
+//
+// Notes:
+//
+int Ls(int argc, wchar_t *argv[])
+{
+  LPWSTR pathName = NULL;
+  LPWSTR longPathName = NULL;
+
+  BY_HANDLE_FILE_INFORMATION fileInformation;
+
+  LPWSTR ownerName = NULL;
+  LPWSTR groupName = NULL;
+  INT unixAccessMode = 0;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  LARGE_INTEGER fileSize;
+
+  BOOL isSymlink = FALSE;
+
+  int ret = EXIT_FAILURE;
+  int optionsMask = 0;
+
+  if (!ParseCommandLine(argc, argv, &pathName, &optionsMask))
+  {
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    LsUsage(argv[0]);
+    return EXIT_FAILURE;
+  }
+
+  assert(pathName != NULL);
+
+  if (wcsspn(pathName, L"/?|><:*\"") != 0)
+  {
+    fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
+    return EXIT_FAILURE;
+  }
+
+  // Convert the path the the long path
+  //
+  dwErrorCode = ConvertToLongPath(pathName, &longPathName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"ConvertToLongPath", dwErrorCode);
+    goto LsEnd;
+  }
+
+  dwErrorCode = GetFileInformationByName(
+    longPathName, optionsMask & CmdLineOptionFollowSymlink, &fileInformation);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"GetFileInformationByName", dwErrorCode);
+    goto LsEnd;
+  }
+
+  dwErrorCode = SymbolicLinkCheck(longPathName, &isSymlink);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+     ReportErrorCode(L"IsSymbolicLink", dwErrorCode);
+     goto LsEnd;
+  }
+
+  if (isSymlink)
+    unixAccessMode |= UX_SYMLINK;
+  else if (IsDirFileInfo(&fileInformation))
+    unixAccessMode |= UX_DIRECTORY;
+
+  dwErrorCode = FindFileOwnerAndPermission(longPathName,
+    &ownerName, &groupName, &unixAccessMode);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"FindFileOwnerAndPermission", dwErrorCode);
+    goto LsEnd;
+  }
+
+  fileSize.HighPart = fileInformation.nFileSizeHigh;
+  fileSize.LowPart = fileInformation.nFileSizeLow;
+
+  // Print output using the input path name (not the long one)
+  //
+  if (!LsPrintLine(unixAccessMode,
+    fileInformation.nNumberOfLinks,
+    ownerName, groupName,
+    &fileInformation.ftLastWriteTime,
+    fileSize,
+    pathName,
+    optionsMask & CmdLineOptionSeparator))
+    goto LsEnd;
+
+  ret = EXIT_SUCCESS;
+
+LsEnd:
+  LocalFree(ownerName);
+  LocalFree(groupName);
+  LocalFree(longPathName);
+
+  return ret;
+}
+
+void LsUsage(LPCWSTR program)
+{
+  fwprintf(stdout, L"\
+Usage: %s [OPTIONS] [FILE]\n\
+List information about the FILE (the current directory by default).\n\
+Using long listing format and list directory entries instead of contents,\n\
+and do not dereference symbolic links.\n\
+Provides equivalent or similar function as 'ls -ld' on GNU/Linux.\n\
+\n\
+OPTIONS: -L dereference symbolic links\n\
+         -F format the output by separating tokens with a pipe\n",
+program);
+}

hadoop-common-project/hadoop-common/src/main/winutils/main.c

@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#include "winutils.h"
+
+static void Usage(LPCWSTR program);
+
+int wmain(int argc, wchar_t* argv[])
+{
+  LPCWSTR cmd = NULL;
+
+  if (argc < 2)
+  {
+    Usage(argv[0]);
+    return EXIT_FAILURE;
+  }
+
+  cmd = argv[1];
+
+  if (wcscmp(L"ls", cmd) == 0)
+  {
+    return Ls(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"chmod", cmd) == 0)
+  {
+    return Chmod(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"chown", cmd) == 0)
+  {
+    return Chown(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"groups", cmd) == 0)
+  {
+    return Groups(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"hardlink", cmd) == 0)
+  {
+    return Hardlink(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"symlink", cmd) == 0)
+  {
+    return Symlink(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"task", cmd) == 0)
+  {
+    return Task(argc - 1, argv + 1);
+  }
+  else if (wcscmp(L"systeminfo", cmd) == 0)
+  {
+    return SystemInfo();
+  }
+  else if (wcscmp(L"help", cmd) == 0)
+  {
+    Usage(argv[0]);
+    return EXIT_SUCCESS;
+  }
+  else
+  {
+    Usage(argv[0]);
+    return EXIT_FAILURE;
+  }
+}
+
+static void Usage(LPCWSTR program)
+{
+  fwprintf(stdout, L"Usage: %s [command] ...\n\
+Provide basic command line utilities for Hadoop on Windows.\n\n\
+The available commands and their usages are:\n\n", program);
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"chmod", L"Change file mode bits.");
+  ChmodUsage(L"chmod");
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"chown", L"Change file owner.");
+  ChownUsage(L"chown");
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"groups", L"List user groups.");
+  GroupsUsage(L"groups");
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"hardlink", L"Hard link operations.");
+  HardlinkUsage();
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"ls", L"List file information.");
+  LsUsage(L"ls");
+  fwprintf(stdout, L"\n\n");
+ 
+  fwprintf(stdout, L"%-10s%s\n\n", L"symlink", L"Create a symbolic link.");
+  SymlinkUsage();
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"systeminfo", L"System information.");
+  SystemInfoUsage();
+  fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations.");
+  TaskUsage();
+  fwprintf(stdout, L"\n\n");
+}

hadoop-common-project/hadoop-common/src/main/winutils/symlink.c

@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+#include "winutils.h"
+
+//----------------------------------------------------------------------------
+// Function: Symlink
+//
+// Description:
+//	The main method for symlink command
+//
+// Returns:
+//	0: on success
+//
+// Notes:
+//
+int Symlink(int argc, wchar_t *argv[])
+{
+  PWSTR longLinkName = NULL;
+  PWSTR longFileName = NULL;
+  DWORD dwErrorCode = ERROR_SUCCESS;
+
+  BOOL isDir = FALSE;
+
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  DWORD dwFlag = 0;
+
+  int ret = SUCCESS;
+
+  if (argc != 3)
+  {
+    SymlinkUsage();
+    return FAILURE;
+  }
+
+  dwErrorCode = ConvertToLongPath(argv[1], &longLinkName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ret = FAILURE;
+    goto SymlinkEnd;
+  }
+  dwErrorCode = ConvertToLongPath(argv[2], &longFileName);
+  if (dwErrorCode != ERROR_SUCCESS)
+  {
+    ret = FAILURE;
+    goto SymlinkEnd;
+  }
+
+  // Check if the the process's access token has the privilege to create
+  // symbolic links. Without this step, the call to CreateSymbolicLink() from
+  // users have the privilege to create symbolic links will still succeed.
+  // This is just an additional step to do the privilege check by not using
+  // error code from CreateSymbolicLink() method.
+  //
+  if (!EnablePrivilege(L"SeCreateSymbolicLinkPrivilege"))
+  {
+    fwprintf(stderr,
+      L"No privilege to create symbolic links.\n");
+    ret = SYMLINK_NO_PRIVILEGE;
+    goto SymlinkEnd;
+  }
+
+  if ((dwRtnCode = DirectoryCheck(longFileName, &isDir)) != ERROR_SUCCESS)
+  {
+    ReportErrorCode(L"DirectoryCheck", dwRtnCode);
+    ret = FAILURE;
+    goto SymlinkEnd;
+  }
+
+  if (isDir)
+    dwFlag = SYMBOLIC_LINK_FLAG_DIRECTORY;
+
+  if (!CreateSymbolicLinkW(longLinkName, longFileName, dwFlag))
+  {
+    ReportErrorCode(L"CreateSymbolicLink", GetLastError());
+    ret = FAILURE;
+    goto SymlinkEnd;
+  }
+
+SymlinkEnd:
+  LocalFree(longLinkName);
+  LocalFree(longFileName);
+  return ret;
+}
+
+void SymlinkUsage()
+{
+    fwprintf(stdout, L"\
+Usage: symlink [LINKNAME] [FILENAME]\n\
+Creates a symbolic link\n\
+\n\
+0 is returned on success.\n\
+2 is returned if the user does no have privilege to create symbolic links.\n\
+1 is returned for all other errors.\n\
+\n\
+The default security settings in Windows disallow non-elevated administrators\n\
+and all non-administrators from creating symbolic links. The security settings\n\
+for symbolic links can be changed in the Local Security Policy management\n\
+console.\n");
+}
+

hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c

@@ -0,0 +1,120 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+* http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include <psapi.h>
+#include <PowrProf.h>
+
+#define PSAPI_VERSION 1
+#pragma comment(lib, "psapi.lib")
+#pragma comment(lib, "Powrprof.lib")
+
+typedef struct _PROCESSOR_POWER_INFORMATION {
+   ULONG  Number;
+   ULONG  MaxMhz;
+   ULONG  CurrentMhz;
+   ULONG  MhzLimit;
+   ULONG  MaxIdleState;
+   ULONG  CurrentIdleState;
+} PROCESSOR_POWER_INFORMATION, *PPROCESSOR_POWER_INFORMATION;
+
+//----------------------------------------------------------------------------
+// Function: SystemInfo
+//
+// Description:
+// Returns the resource information about the machine 
+//
+// Returns:
+// EXIT_SUCCESS: On success
+// EXIT_FAILURE: otherwise
+int SystemInfo() 
+{
+  size_t vmemSize, vmemFree, memSize, memFree;
+  PERFORMANCE_INFORMATION memInfo;
+  SYSTEM_INFO sysInfo;
+  FILETIME idleTimeFt, kernelTimeFt, userTimeFt;
+  ULARGE_INTEGER idleTime, kernelTime, userTime;
+  ULONGLONG cpuTimeMs;
+  size_t size;
+  LPBYTE pBuffer;
+  PPROCESSOR_POWER_INFORMATION ppi;
+  long cpuFrequencyKhz;
+  NTSTATUS status;
+
+  ZeroMemory(&memInfo, sizeof(PERFORMANCE_INFORMATION));
+  memInfo.cb = sizeof(PERFORMANCE_INFORMATION);
+  if(!GetPerformanceInfo(&memInfo, sizeof(PERFORMANCE_INFORMATION)))
+  {
+    ReportErrorCode(L"GetPerformanceInfo", GetLastError());
+    return EXIT_FAILURE;
+  }
+  vmemSize = memInfo.CommitLimit*memInfo.PageSize;
+  vmemFree = vmemSize - memInfo.CommitTotal*memInfo.PageSize;
+  memSize = memInfo.PhysicalTotal*memInfo.PageSize;
+  memFree = memInfo.PhysicalAvailable*memInfo.PageSize;
+
+  GetSystemInfo(&sysInfo);
+
+  if(!GetSystemTimes(&idleTimeFt, &kernelTimeFt, &userTimeFt))
+  {
+    ReportErrorCode(L"GetSystemTimes", GetLastError());
+    return EXIT_FAILURE;
+  }
+  idleTime.HighPart = idleTimeFt.dwHighDateTime;
+  idleTime.LowPart = idleTimeFt.dwLowDateTime;
+  kernelTime.HighPart = kernelTimeFt.dwHighDateTime;
+  kernelTime.LowPart = kernelTimeFt.dwLowDateTime;
+  userTime.HighPart = userTimeFt.dwHighDateTime;
+  userTime.LowPart = userTimeFt.dwLowDateTime;
+
+  cpuTimeMs = (kernelTime.QuadPart - idleTime.QuadPart + userTime.QuadPart)/10000;
+
+  // allocate buffer to get info for each processor
+  size = sysInfo.dwNumberOfProcessors * sizeof(PROCESSOR_POWER_INFORMATION);
+  pBuffer = (BYTE*) LocalAlloc(LPTR, size);
+  if(!pBuffer)
+  {
+    ReportErrorCode(L"LocalAlloc", GetLastError());
+    return EXIT_FAILURE;
+  }
+  status = CallNtPowerInformation(ProcessorInformation, NULL, 0, pBuffer, (long)size);
+  if(0 != status)
+  {
+    fwprintf_s(stderr, L"Error in CallNtPowerInformation. Err:%d\n", status);
+    LocalFree(pBuffer);
+    return EXIT_FAILURE;
+  }
+  ppi = (PPROCESSOR_POWER_INFORMATION)pBuffer;
+  cpuFrequencyKhz = ppi->MaxMhz*1000;
+  LocalFree(pBuffer);
+
+  fwprintf_s(stdout, L"%Iu,%Iu,%Iu,%Iu,%Iu,%Iu,%Iu\n", vmemSize, memSize, vmemFree, memFree, sysInfo.dwNumberOfProcessors, cpuFrequencyKhz, cpuTimeMs);
+
+  return EXIT_SUCCESS;
+}
+
+void SystemInfoUsage()
+{
+    fwprintf(stdout, L"\
+    Usage: systeminfo\n\
+    Prints machine information on stdout\n\
+    Comma separated list of the following values.\n\
+    VirtualMemorySize(bytes),PhysicalMemorySize(bytes),\n\
+    FreeVirtualMemory(bytes),FreePhysicalMemory(bytes),\n\
+    NumberOfProcessors,CpuFrequency(Khz),\n\
+    CpuTime(MilliSec,Kernel+User)\n");
+}

hadoop-common-project/hadoop-common/src/main/winutils/task.c

@@ -0,0 +1,461 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+* http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include <errno.h>
+#include <psapi.h>
+
+#define PSAPI_VERSION 1
+#pragma comment(lib, "psapi.lib")
+
+#define ERROR_TASK_NOT_ALIVE 1
+
+// List of different task related command line options supported by
+// winutils.
+typedef enum TaskCommandOptionType
+{
+  TaskInvalid,
+  TaskCreate,
+  TaskIsAlive,
+  TaskKill,
+  TaskProcessList
+} TaskCommandOption;
+
+//----------------------------------------------------------------------------
+// Function: ParseCommandLine
+//
+// Description:
+//  Parses the given command line. On success, out param 'command' contains
+//  the user specified command.
+//
+// Returns:
+// TRUE: If the command line is valid
+// FALSE: otherwise
+static BOOL ParseCommandLine(__in int argc,
+                             __in wchar_t *argv[],
+                             __out TaskCommandOption *command)
+{
+  *command = TaskInvalid;
+
+  if (wcscmp(argv[0], L"task") != 0 )
+  {
+    return FALSE;
+  }
+
+  if (argc == 3) {
+    if (wcscmp(argv[1], L"isAlive") == 0)
+    {
+      *command = TaskIsAlive;
+      return TRUE;
+    }
+    if (wcscmp(argv[1], L"kill") == 0)
+    {
+      *command = TaskKill;
+      return TRUE;
+    }
+    if (wcscmp(argv[1], L"processList") == 0)
+    {
+      *command = TaskProcessList;
+      return TRUE;
+    }
+  }
+
+  if (argc == 4) {
+    if (wcscmp(argv[1], L"create") == 0)
+    {
+      *command = TaskCreate;
+      return TRUE;
+    }
+  }
+
+  return FALSE;
+}
+
+//----------------------------------------------------------------------------
+// Function: createTask
+//
+// Description:
+//  Creates a task via a jobobject. Outputs the
+//  appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD createTask(_TCHAR* jobObjName, _TCHAR* cmdLine) 
+{
+  DWORD err = ERROR_SUCCESS;
+  DWORD exitCode = EXIT_FAILURE;
+  STARTUPINFO si;
+  PROCESS_INFORMATION pi;
+  HANDLE jobObject = NULL;
+  JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 };
+
+  // Create un-inheritable job object handle and set job object to terminate 
+  // when last handle is closed. So winutils.exe invocation has the only open 
+  // job object handle. Exit of winutils.exe ensures termination of job object.
+  // Either a clean exit of winutils or crash or external termination.
+  jobObject = CreateJobObject(NULL, jobObjName);
+  err = GetLastError();
+  if(jobObject == NULL || err ==  ERROR_ALREADY_EXISTS)
+  {
+    return err;
+  }
+  jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+  if(SetInformationJobObject(jobObject, 
+                             JobObjectExtendedLimitInformation, 
+                             &jeli, 
+                             sizeof(jeli)) == 0)
+  {
+    err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }      
+
+  if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0)
+  {
+    err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+
+  // the child JVM uses this env var to send the task OS process identifier 
+  // to the TaskTracker. We pass the job object name.
+  if(SetEnvironmentVariable(_T("JVM_PID"), jobObjName) == 0)
+  {
+    err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+
+  ZeroMemory( &si, sizeof(si) );
+  si.cb = sizeof(si);
+  ZeroMemory( &pi, sizeof(pi) );
+  if(CreateProcess(NULL, cmdLine, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi) == 0)
+  {
+    err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+  CloseHandle(pi.hThread);
+
+  // Wait until child process exits.
+  WaitForSingleObject( pi.hProcess, INFINITE );
+  if(GetExitCodeProcess(pi.hProcess, &exitCode) == 0)
+  {
+    err = GetLastError();
+  }
+  CloseHandle( pi.hProcess );
+
+  // Terminate job object so that all spawned processes are also killed.
+  // This is needed because once this process closes the handle to the job 
+  // object and none of the spawned objects have the handle open (via 
+  // inheritance on creation) then it will not be possible for any other external 
+  // program (say winutils task kill) to terminate this job object via its name.
+  if(TerminateJobObject(jobObject, exitCode) == 0)
+  {
+    err = GetLastError();
+  }
+
+  // comes here only on failure or TerminateJobObject
+  CloseHandle(jobObject);
+
+  if(err != ERROR_SUCCESS)
+  {
+    return err;
+  }
+  return exitCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: isTaskAlive
+//
+// Description:
+//  Checks if a task is alive via a jobobject. Outputs the
+//  appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD isTaskAlive(const _TCHAR* jobObjName, int* isAlive, int* procsInJob)
+{
+  PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
+  HANDLE jobObject = NULL;
+  int numProcs = 100;
+
+  *isAlive = FALSE;
+  
+  jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
+
+  if(jobObject == NULL)
+  {
+    DWORD err = GetLastError();
+    if(err == ERROR_FILE_NOT_FOUND)
+    {
+      // job object does not exist. assume its not alive
+      return ERROR_SUCCESS;
+    }
+    return err;
+  }
+
+  procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32);
+  if (!procList)
+  {
+    DWORD err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+  if(QueryInformationJobObject(jobObject, JobObjectBasicProcessIdList, procList, sizeof(JOBOBJECT_BASIC_PROCESS_ID_LIST)+numProcs*32, NULL) == 0)
+  {
+    DWORD err = GetLastError();
+    if(err != ERROR_MORE_DATA) 
+    {
+      CloseHandle(jobObject);
+      LocalFree(procList);
+      return err;
+    }
+  }
+
+  if(procList->NumberOfAssignedProcesses > 0)
+  {
+    *isAlive = TRUE;
+    *procsInJob = procList->NumberOfAssignedProcesses;
+  }
+
+  LocalFree(procList);
+
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: killTask
+//
+// Description:
+//  Kills a task via a jobobject. Outputs the
+//  appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD killTask(_TCHAR* jobObjName)
+{
+  HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName);
+  if(jobObject == NULL)
+  {
+    DWORD err = GetLastError();
+    if(err == ERROR_FILE_NOT_FOUND)
+    {
+      // job object does not exist. assume its not alive
+      return ERROR_SUCCESS;
+    }
+    return err;
+  }
+
+  if(TerminateJobObject(jobObject, 1) == 0)
+  {
+    return GetLastError();
+  }
+  CloseHandle(jobObject);
+
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: printTaskProcessList
+//
+// Description:
+// Prints resource usage of all processes in the task jobobject
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD printTaskProcessList(const _TCHAR* jobObjName)
+{
+  DWORD i;
+  PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
+  int numProcs = 100;
+  HANDLE jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
+  if(jobObject == NULL)
+  {
+    DWORD err = GetLastError();
+    return err;
+  }
+
+  procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32);
+  if (!procList)
+  {
+    DWORD err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+  while(QueryInformationJobObject(jobObject, JobObjectBasicProcessIdList, procList, sizeof(JOBOBJECT_BASIC_PROCESS_ID_LIST)+numProcs*32, NULL) == 0)
+  {
+    DWORD err = GetLastError();
+    if(err != ERROR_MORE_DATA) 
+    {
+      CloseHandle(jobObject);
+      LocalFree(procList);
+      return err;
+    }
+    numProcs = procList->NumberOfAssignedProcesses;
+    LocalFree(procList);
+    procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32);
+    if (!procList)
+    {
+      DWORD err = GetLastError();
+      CloseHandle(jobObject);
+      return err;
+    }
+  }
+
+  for(i=0; i<procList->NumberOfProcessIdsInList; ++i)
+  {
+    HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION, FALSE, (DWORD)procList->ProcessIdList[i] );
+    if( hProcess != NULL )
+    {
+      PROCESS_MEMORY_COUNTERS_EX pmc;
+      if ( GetProcessMemoryInfo( hProcess, (PPROCESS_MEMORY_COUNTERS)&pmc, sizeof(pmc)) )
+      {
+        FILETIME create, exit, kernel, user;
+        if( GetProcessTimes( hProcess, &create, &exit, &kernel, &user) )
+        {
+          ULARGE_INTEGER kernelTime, userTime;
+          ULONGLONG cpuTimeMs;
+          kernelTime.HighPart = kernel.dwHighDateTime;
+          kernelTime.LowPart = kernel.dwLowDateTime;
+          userTime.HighPart = user.dwHighDateTime;
+          userTime.LowPart = user.dwLowDateTime;
+          cpuTimeMs = (kernelTime.QuadPart+userTime.QuadPart)/10000;
+          _ftprintf_s(stdout, TEXT("%u,%Iu,%Iu,%Iu\n"), procList->ProcessIdList[i], pmc.PrivateUsage, pmc.WorkingSetSize, cpuTimeMs);
+        }
+      }
+      CloseHandle( hProcess );
+    }
+  }
+
+  LocalFree(procList);
+  CloseHandle(jobObject);
+
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: Task
+//
+// Description:
+//  Manages a task via a jobobject (create/isAlive/kill). Outputs the
+//  appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// Error code otherwise: otherwise
+int Task(int argc, wchar_t *argv[])
+{
+  DWORD dwErrorCode = ERROR_SUCCESS;
+  TaskCommandOption command = TaskInvalid;
+
+  if (!ParseCommandLine(argc, argv, &command)) {
+    dwErrorCode = ERROR_INVALID_COMMAND_LINE;
+
+    fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+    TaskUsage();
+    goto TaskExit;
+  }
+
+  if (command == TaskCreate)
+  {
+    // Create the task jobobject
+    //
+    dwErrorCode = createTask(argv[2], argv[3]);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"createTask", dwErrorCode);
+      goto TaskExit;
+    }
+  } else if (command == TaskIsAlive)
+  {
+    // Check if task jobobject
+    //
+    int isAlive;
+    int numProcs;
+    dwErrorCode = isTaskAlive(argv[2], &isAlive, &numProcs);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"isTaskAlive", dwErrorCode);
+      goto TaskExit;
+    }
+
+    // Output the result
+    if(isAlive == TRUE)
+    {
+      fwprintf(stdout, L"IsAlive,%d\n", numProcs);
+    }
+    else
+    {
+      dwErrorCode = ERROR_TASK_NOT_ALIVE;
+      ReportErrorCode(L"isTaskAlive returned false", dwErrorCode);
+      goto TaskExit;
+    }
+  } else if (command == TaskKill)
+  {
+    // Check if task jobobject
+    //
+    dwErrorCode = killTask(argv[2]);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"killTask", dwErrorCode);
+      goto TaskExit;
+    }
+  } else if (command == TaskProcessList)
+  {
+    // Check if task jobobject
+    //
+    dwErrorCode = printTaskProcessList(argv[2]);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"printTaskProcessList", dwErrorCode);
+      goto TaskExit;
+    }
+  } else
+  {
+    // Should not happen
+    //
+    assert(FALSE);
+  }
+
+TaskExit:
+  return dwErrorCode;
+}
+
+void TaskUsage()
+{
+  // Hadoop code checks for this string to determine if
+  // jobobject's are being used.
+  // ProcessTree.isSetsidSupported()
+  fwprintf(stdout, L"\
+    Usage: task create [TASKNAME] [COMMAND_LINE] |\n\
+          task isAlive [TASKNAME] |\n\
+          task kill [TASKNAME]\n\
+          task processList [TASKNAME]\n\
+    Creates a new task jobobject with taskname\n\
+    Checks if task jobobject is alive\n\
+    Kills task jobobject\n\
+    Prints to stdout a list of processes in the task\n\
+    along with their resource usage. One process per line\n\
+    and comma separated info per process\n\
+    ProcessId,VirtualMemoryCommitted(bytes),\n\
+    WorkingSetSize(bytes),CpuTime(Millisec,Kernel+User)\n");
+}

hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln

@@ -0,0 +1,55 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}"
+	ProjectSection(ProjectDependencies) = postProject
+		{12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12}
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils.vcxproj", "{12131AA7-902E-4A6D-9CE3-043261D22A12}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64
+		{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64
+		{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj

@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>winutils</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IncludePath>include;$(IncludePath)</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <OutDir />
+    <IntDir>..\..\..\target\winutils\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IntDir>..\..\..\target\winutils\$(Platform)\$(Configuration)\</IntDir>
+    <OutDir>..\..\..\target\bin\</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="symlink.c" />
+    <ClCompile Include="systeminfo.c" />
+    <ClCompile Include="chmod.c" />
+    <ClCompile Include="chown.c" />
+    <ClCompile Include="groups.c" />
+    <ClCompile Include="hardlink.c" />
+    <ClCompile Include="task.c" />
+    <ClCompile Include="ls.c" />
+    <ClCompile Include="main.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="libwinutils.vcxproj">
+      <Project>{12131aa7-902e-4a6d-9ce3-043261d22a12}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm

@@ -33,9 +33,7 @@ Single Node Setup
      * GNU/Linux is supported as a development and production platform.
        Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
 
-     * Win32 is supported as a development platform. Distributed operation
-       has not been well tested on Win32, so it is not supported as a
-       production platform.
+     * Windows is also a supported platform.
 
 ** Required Software
 
@@ -46,11 +44,6 @@ Single Node Setup
     [[2]] ssh must be installed and sshd must be running to use the Hadoop
        scripts that manage remote Hadoop daemons.
 
-   Additional requirements for Windows include:
-
-    [[1]] Cygwin - Required for shell support in addition to the required
-       software above.
-
 ** Installing Software
 
    If your cluster doesn't have the requisite software you will need to
@@ -63,11 +56,6 @@ Single Node Setup
    $ sudo apt-get install rsync
 ----
 
-   On Windows, if you did not install the required software when you
-   installed cygwin, start the cygwin installer and select the packages:
-
-     * openssh - the Net category
-
 * Download
 
    To get a Hadoop distribution, download a recent stable release from one