HADOOP-1570. Permit jobs to enable and disable use of Hadoop's native library. Contributed by Arun.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@555768 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -343,6 +343,9 @@ Trunk (unreleased changes)
 106. HADOOP-1547.  Provide examples for aggregate library.
      (Runping Qi via tomwhite)
 
+107. HADOOP-1570.  Permit jobs to enable and disable the use of
+     hadoop's native library.  (Arun C Murthy via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

conf/hadoop-default.xml

@@ -15,6 +15,11 @@
   <description>A base for other temporary directories.</description>
 </property>
 
+<property>
+  <name>hadoop.native.lib</name>
+  <value>true</value>
+  <description>Should native hadoop libraries, if present, be used.</description>
+</property>
 
 <!--- logging properties -->
 

src/java/org/apache/hadoop/io/SequenceFile.java

@@ -193,7 +193,7 @@ public class SequenceFile {
     throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -234,7 +234,7 @@ public class SequenceFile {
                  Progressable progress, Metadata metadata) throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -295,7 +295,7 @@ public class SequenceFile {
     throws IOException {
     if (codec != null && (codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }
@@ -336,7 +336,7 @@ public class SequenceFile {
   throws IOException {
   if (codec != null && (codec instanceof GzipCodec) && 
       !NativeCodeLoader.isNativeCodeLoaded() && 
-      !ZlibFactory.isNativeZlibLoaded()) {
+      !ZlibFactory.isNativeZlibLoaded(conf)) {
     throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                        "GzipCodec without native-hadoop code!");
   }
@@ -375,7 +375,7 @@ public class SequenceFile {
     throws IOException {
     if ((codec instanceof GzipCodec) && 
         !NativeCodeLoader.isNativeCodeLoaded() && 
-        !ZlibFactory.isNativeZlibLoaded()) {
+        !ZlibFactory.isNativeZlibLoaded(conf)) {
       throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                          "GzipCodec without native-hadoop code!");
     }

src/java/org/apache/hadoop/io/compress/DefaultCodec.java

@@ -52,11 +52,11 @@ public class DefaultCodec implements Configurable, CompressionCodec {
   }
 
   public Class getCompressorType() {
-    return ZlibFactory.getZlibCompressorType();
+    return ZlibFactory.getZlibCompressorType(conf);
   }
 
   public Compressor createCompressor() {
-    return ZlibFactory.getZlibCompressor();
+    return ZlibFactory.getZlibCompressor(conf);
   }
 
   public CompressionInputStream createInputStream(InputStream in) 
@@ -73,11 +73,11 @@ public class DefaultCodec implements Configurable, CompressionCodec {
   }
 
   public Class getDecompressorType() {
-    return ZlibFactory.getZlibDecompressorType();
+    return ZlibFactory.getZlibDecompressorType(conf);
   }
 
   public Decompressor createDecompressor() {
-    return ZlibFactory.getZlibDecompressor();
+    return ZlibFactory.getZlibDecompressor(conf);
   }
   
   public String getDefaultExtension() {

src/java/org/apache/hadoop/io/compress/GzipCodec.java

@@ -135,7 +135,7 @@ public class GzipCodec extends DefaultCodec {
   
   public CompressionOutputStream createOutputStream(OutputStream out) 
     throws IOException {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new CompressorStream(out, createCompressor(),
                                     conf.getInt("io.file.buffer.size", 4*1024)) :
                new GzipOutputStream(out);
@@ -153,7 +153,7 @@ public class GzipCodec extends DefaultCodec {
   }
 
   public Compressor createCompressor() {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new ZlibCompressor(ZlibCompressor.CompressionLevel.DEFAULT_COMPRESSION,
                                   ZlibCompressor.CompressionStrategy.DEFAULT_STRATEGY,
                                   ZlibCompressor.CompressionHeader.GZIP_FORMAT,
@@ -162,12 +162,12 @@ public class GzipCodec extends DefaultCodec {
   }
 
   public Class getCompressorType() {
-    return ZlibFactory.getZlibCompressorType();
+    return ZlibFactory.getZlibCompressorType(conf);
   }
 
   public CompressionInputStream createInputStream(InputStream in) 
   throws IOException {
-  return (ZlibFactory.isNativeZlibLoaded()) ?
+  return (ZlibFactory.isNativeZlibLoaded(conf)) ?
              new DecompressorStream(in, createDecompressor(),
                                     conf.getInt("io.file.buffer.size", 
                                                 4*1024)) :
@@ -185,14 +185,14 @@ public class GzipCodec extends DefaultCodec {
   }
 
   public Decompressor createDecompressor() {
-    return (ZlibFactory.isNativeZlibLoaded()) ?
+    return (ZlibFactory.isNativeZlibLoaded(conf)) ?
                new ZlibDecompressor(ZlibDecompressor.CompressionHeader.AUTODETECT_GZIP_ZLIB,
                                     64*1024) :
                null;                               
   }
 
   public Class getDecompressorType() {
-    return ZlibFactory.getZlibDecompressorType();
+    return ZlibFactory.getZlibDecompressorType(conf);
   }
 
   public String getDefaultExtension() {

src/java/org/apache/hadoop/io/compress/LzoCodec.java

@@ -69,17 +69,18 @@ public class LzoCodec implements Configurable, CompressionCodec {
   /**
    * Check if native-lzo library is loaded & initialized.
    * 
+   * @param conf configuration
    * @return <code>true</code> if native-lzo library is loaded & initialized;
    *         else <code>false</code>
    */
-  public static boolean isNativeLzoLoaded() {
-    return nativeLzoLoaded;
+  public static boolean isNativeLzoLoaded(Configuration conf) {
+    return nativeLzoLoaded && conf.getBoolean("hadoop.native.lib", true);
   }
   
   public CompressionOutputStream createOutputStream(OutputStream out) 
     throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -128,7 +129,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
                                                     Compressor compressor) 
   throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -153,7 +154,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
 
   public Class getCompressorType() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -162,7 +163,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
 
   public Compressor createCompressor() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -181,7 +182,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
   public CompressionInputStream createInputStream(InputStream in) 
     throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new IOException("native-lzo library not available");
     }
     
@@ -204,7 +205,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
                                                   Decompressor decompressor) 
   throws IOException {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -215,7 +216,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
 
   public Class getDecompressorType() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     
@@ -224,7 +225,7 @@ public class LzoCodec implements Configurable, CompressionCodec {
 
   public Decompressor createDecompressor() {
     // Ensure native-lzo library is loaded & initialized
-    if (!isNativeLzoLoaded()) {
+    if (!isNativeLzoLoaded(conf)) {
       throw new RuntimeException("native-lzo library not available");
     }
     

src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java

@@ -20,10 +20,10 @@ package org.apache.hadoop.io.compress.zlib;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.Decompressor;
 import org.apache.hadoop.util.NativeCodeLoader;
-import org.apache.hadoop.util.StringUtils;
 
 /**
  * A collection of factories to create the right 
@@ -50,54 +50,58 @@ public class ZlibFactory {
   }
   
   /**
-   * Check if native-zlib code is loaded and initialized correctly.
+   * Check if native-zlib code is loaded & initialized correctly and 
+   * can be loaded for this job.
    * 
-   * @return <code>true</code> if native-zlib is loaded and initialized, 
-   *         else <code>false</code>
+   * @param conf configuration
+   * @return <code>true</code> if native-zlib is loaded & initialized 
+   *         and can be loaded for this job, else <code>false</code>
    */
-  public static boolean isNativeZlibLoaded() {
-    return nativeZlibLoaded; 
+  public static boolean isNativeZlibLoaded(Configuration conf) {
+    return nativeZlibLoaded && conf.getBoolean("hadoop.native.lib", true); 
   }
   
   /**
    * Return the appropriate type of the zlib compressor. 
    * 
+   * @param conf configuration
    * @return the appropriate type of the zlib compressor.
    */
-  public static Class getZlibCompressorType() {
-    return (nativeZlibLoaded) ? 
+  public static Class getZlibCompressorType(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
             ZlibCompressor.class : BuiltInZlibDeflater.class;
   }
   
   /**
    * Return the appropriate implementation of the zlib compressor. 
    * 
+   * @param conf configuration
    * @return the appropriate implementation of the zlib compressor.
    */
-  public static Compressor getZlibCompressor() {
-    LOG.info("Creating a new ZlibCompressor");
-    return (nativeZlibLoaded) ? 
+  public static Compressor getZlibCompressor(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
       new ZlibCompressor() : new BuiltInZlibDeflater(); 
   }
 
   /**
    * Return the appropriate type of the zlib decompressor. 
    * 
+   * @param conf configuration
    * @return the appropriate type of the zlib decompressor.
    */
-  public static Class getZlibDecompressorType() {
-    return (nativeZlibLoaded) ? 
+  public static Class getZlibDecompressorType(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
             ZlibDecompressor.class : BuiltInZlibInflater.class;
   }
   
   /**
    * Return the appropriate implementation of the zlib decompressor. 
    * 
+   * @param conf configuration
    * @return the appropriate implementation of the zlib decompressor.
    */
-  public static Decompressor getZlibDecompressor() {
-    LOG.info("Creating a new ZlibDecompressor");
-    return (nativeZlibLoaded) ? 
+  public static Decompressor getZlibDecompressor(Configuration conf) {
+    return (isNativeZlibLoaded(conf)) ? 
       new ZlibDecompressor() : new BuiltInZlibInflater(); 
   }
   

src/java/org/apache/hadoop/util/NativeCodeLoader.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.util;
 
 import org.apache.commons.logging.*;
+import org.apache.hadoop.mapred.JobConf;
 
 /**
  * A helper to load the native hadoop code i.e. libhadoop.so.
@@ -62,4 +63,26 @@ public class NativeCodeLoader {
     return nativeCodeLoaded;
   }
 
+  /**
+   * Return if native hadoop libraries, if present, can be used for this job.
+   * @param jobConf job configuration
+   * 
+   * @return <code>true</code> if native hadoop libraries, if present, can be 
+   *         used for this job; <code>false</code> otherwise.
+   */
+  public boolean getLoadNativeLibraries(JobConf jobConf) {
+    return jobConf.getBoolean("hadoop.native.lib", true);
+  }
+  
+  /**
+   * Set if native hadoop libraries, if present, can be used for this job.
+   * 
+   * @param jobConf job configuration
+   * @param loadNativeLibraries can native hadoop libraries be loaded
+   */
+  public void setLoadNativeLibraries(JobConf jobConf, 
+                                     boolean loadNativeLibraries) {
+    jobConf.setBoolean("hadoop.native.lib", loadNativeLibraries);
+  }
+
 }

src/test/org/apache/hadoop/io/TestSequenceFile.java

@@ -49,7 +49,7 @@ public class TestSequenceFile extends TestCase {
   }
   
   public void testLzoSequenceFile() throws Exception {
-    if (LzoCodec.isNativeLzoLoaded()) {
+    if (LzoCodec.isNativeLzoLoaded(conf)) {
       LOG.info("Testing SequenceFile with LzoCodec");
       CompressionCodec lzoCodec = null;
       try {

src/test/org/apache/hadoop/io/compress/TestCodec.java

@@ -40,28 +40,29 @@ public class TestCodec extends TestCase {
   private static final Log LOG= 
     LogFactory.getLog("org.apache.hadoop.io.compress.TestCodec");
 
+  private Configuration conf = new Configuration();
   private int count = 10000;
   private int seed = new Random().nextInt();
   
   public void testDefaultCodec() throws IOException {
-    codecTest(seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
+    codecTest(conf, seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
   }
   
   public void testGzipCodec() throws IOException {
-    codecTest(seed, count, "org.apache.hadoop.io.compress.GzipCodec");
+    codecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec");
   }
   
   public void testLzoCodec() throws IOException {
-    if (LzoCodec.isNativeLzoLoaded()) {
-      codecTest(seed, count, "org.apache.hadoop.io.compress.LzoCodec");
+    if (LzoCodec.isNativeLzoLoaded(conf)) {
+      codecTest(conf, seed, count, "org.apache.hadoop.io.compress.LzoCodec");
     }
   }
   
-  private static void codecTest(int seed, int count, String codecClass) 
+  private static void codecTest(Configuration conf, int seed, int count, 
+                                String codecClass) 
     throws IOException {
     
     // Create the codec
-    Configuration conf = new Configuration();
     CompressionCodec codec = null;
     try {
       codec = (CompressionCodec)
@@ -144,8 +145,9 @@ public class TestCodec extends TestCase {
         }
       }
 
+      Configuration conf = new Configuration();
       int seed = 0;
-      codecTest(seed, count, codecClass);
+      codecTest(conf, seed, count, codecClass);
     } catch (Exception e) {
       System.err.println("Caught: " + e);
       e.printStackTrace();