HADOOP-11057. checknative command to probe for winutils.exe on windows. Contributed by Xiaoyu Yao.

(cherry picked from commit 6dae4b430c342f9ad44ad8659c372e519f3931c9)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -170,6 +170,9 @@ Release 2.6.0 - UNRELEASED
 
     HADOOP-11070. Create MiniKMS for testing. (tucu)
 
+    HADOOP-11057. checknative command to probe for winutils.exe on windows.
+    (Xiaoyu Yao via cnauroth)
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java

@@ -37,7 +37,8 @@ public class NativeLibraryChecker {
   public static void main(String[] args) {
     String usage = "NativeLibraryChecker [-a|-h]\n"
         + "  -a  use -a to check all libraries are available\n"
-        + "      by default just check hadoop library is available\n"
+        + "      by default just check hadoop library (and\n"
+        + "      winutils.exe on Windows OS) is available\n"
         + "      exit with error code 1 if check failed\n"
         + "  -h  print this message\n";
     if (args.length > 1 ||
@@ -62,12 +63,16 @@ public class NativeLibraryChecker {
     boolean lz4Loaded = nativeHadoopLoaded;
     boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
     boolean openSslLoaded = false;
+    boolean winutilsExists = false;
+
     String openSslDetail = "";
     String hadoopLibraryName = "";
     String zlibLibraryName = "";
     String snappyLibraryName = "";
     String lz4LibraryName = "";
     String bzip2LibraryName = "";
+    String winutilsPath = null;
+
     if (nativeHadoopLoaded) {
       hadoopLibraryName = NativeCodeLoader.getLibraryName();
       zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
@@ -93,6 +98,15 @@ public class NativeLibraryChecker {
         bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
       }
     }
+
+    // winutils.exe is required on Windows
+    winutilsPath = Shell.getWinUtilsPath();
+    if (winutilsPath != null) {
+      winutilsExists = true;
+    } else {
+      winutilsPath = "";
+    }
+
     System.out.println("Native library checking:");
     System.out.printf("hadoop:  %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
     System.out.printf("zlib:    %b %s\n", zlibLoaded, zlibLibraryName);
@@ -100,7 +114,11 @@ public class NativeLibraryChecker {
     System.out.printf("lz4:     %b %s\n", lz4Loaded, lz4LibraryName);
     System.out.printf("bzip2:   %b %s\n", bzip2Loaded, bzip2LibraryName);
     System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
-    if ((!nativeHadoopLoaded) ||
+    if (Shell.WINDOWS) {
+      System.out.printf("winutils: %b %s\n", winutilsExists, winutilsPath);
+    }
+
+    if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
         (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
       // return 1 to indicated check failed
       ExitUtil.terminate(1);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java

@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.util;
 
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
 import junit.framework.TestCase;
 
 import org.apache.hadoop.util.ExitUtil.ExitException;
@@ -51,4 +54,30 @@ public class TestNativeLibraryChecker extends TestCase {
     }
   }
 
+  @Test
+  public void testNativeLibraryCheckerOutput(){
+    expectOutput(new String[]{"-a"});
+    // no argument
+    expectOutput(new String[0]);
+  }
+
+  private void expectOutput(String [] args) {
+    ExitUtil.disableSystemExit();
+    ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    PrintStream originalPs = System.out;
+    System.setOut(new PrintStream(outContent));
+    try {
+      NativeLibraryChecker.main(args);
+    } catch (ExitException e) {
+      ExitUtil.resetFirstExitException();
+    } finally {
+      if (Shell.WINDOWS) {
+        assertEquals(outContent.toString().indexOf("winutils: true") != -1, true);
+      }
+      if (NativeCodeLoader.isNativeCodeLoaded()) {
+        assertEquals(outContent.toString().indexOf("hadoop:  true") != -1, true);
+      }
+      System.setOut(originalPs);
+    }
+  }
 }