HADOOP-13665. Erasure Coding codec should support fallback coder. Contributed by Kai Sasaki.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java

@@ -18,6 +18,10 @@
 package org.apache.hadoop.io.erasurecode;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.erasurecode.codec.ErasureCodec;
@@ -26,6 +30,8 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
 import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
 import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
 import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
@@ -35,6 +41,7 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
 
 /**
  * A codec & coder utility to help create coders conveniently.
@@ -49,41 +56,50 @@ import java.lang.reflect.InvocationTargetException;
 @InterfaceAudience.Private
 public final class CodecUtil {
 
+  private static final Log LOG = LogFactory.getLog(CodecUtil.class);
+
+  public static final String IO_ERASURECODE_CODEC = "io.erasurecode.codec.";
+
   /** Erasure coder XOR codec. */
   public static final String IO_ERASURECODE_CODEC_XOR_KEY =
-      "io.erasurecode.codec.xor";
+      IO_ERASURECODE_CODEC + "xor";
   public static final String IO_ERASURECODE_CODEC_XOR =
       XORErasureCodec.class.getCanonicalName();
   /** Erasure coder Reed-Solomon codec. */
   public static final String IO_ERASURECODE_CODEC_RS_KEY =
-      "io.erasurecode.codec.rs";
+      IO_ERASURECODE_CODEC + "rs";
   public static final String IO_ERASURECODE_CODEC_RS =
       RSErasureCodec.class.getCanonicalName();
   /** Erasure coder hitch hiker XOR codec. */
   public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
-      "io.erasurecode.codec.hhxor";
+      IO_ERASURECODE_CODEC + "hhxor";
   public static final String IO_ERASURECODE_CODEC_HHXOR =
       HHXORErasureCodec.class.getCanonicalName();
 
-  /** Supported erasure codec classes. */
-
-  /** Raw coder factory for the RS codec. */
-  public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
-      "io.erasurecode.codec.rs.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT =
-      RSRawErasureCoderFactory.class.getCanonicalName();
-
-  /** Raw coder factory for the RS legacy codec. */
-  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
-      "io.erasurecode.codec.rs-legacy.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
+  /** Comma separated raw codec name. The first coder is prior to the latter. */
+  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
+      IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
       RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
+  public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
+      IO_ERASURECODE_CODEC + "rs.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
+      NativeRSRawErasureCoderFactory.class.getCanonicalName() +
+      "," + RSRawErasureCoderFactory.class.getCanonicalName();
 
   /** Raw coder factory for the XOR codec. */
-  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
-      "io.erasurecode.codec.xor.rawcoder";
-  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
-      XORRawErasureCoderFactory.class.getCanonicalName();
+  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
+      IO_ERASURECODE_CODEC + "xor.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT =
+      NativeXORRawErasureCoderFactory.class.getCanonicalName() +
+      "," + XORRawErasureCoderFactory.class.getCanonicalName();
+
+  // Default coders for each codec names.
+  public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of(
+      "rs",         IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT,
+      "rs-legacy",  IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT,
+      "xor",        IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT
+  );
 
   private CodecUtil() { }
 
@@ -133,12 +149,7 @@ public final class CodecUtil {
     Preconditions.checkNotNull(conf);
     Preconditions.checkNotNull(codec);
 
-    String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
-
-    RawErasureCoderFactory fact = createRawCoderFactory(conf,
-        rawCoderFactoryKey);
-
-    return fact.createEncoder(coderOptions);
+    return createRawEncoderWithFallback(conf, codec, coderOptions);
   }
 
   /**
@@ -153,12 +164,7 @@ public final class CodecUtil {
     Preconditions.checkNotNull(conf);
     Preconditions.checkNotNull(codec);
 
-    String rawCoderFactoryKey = getRawCoderFactNameFromCodec(conf, codec);
-
-    RawErasureCoderFactory fact = createRawCoderFactory(conf,
-        rawCoderFactoryKey);
-
-    return fact.createDecoder(coderOptions);
+    return createRawDecoderWithFallback(conf, codec, coderOptions);
   }
 
   private static RawErasureCoderFactory createRawCoderFactory(
@@ -180,31 +186,52 @@ public final class CodecUtil {
     return fact;
   }
 
-  private static String getRawCoderFactNameFromCodec(Configuration conf,
-                                                     String codec) {
-    switch (codec) {
-    case ErasureCodeConstants.RS_CODEC_NAME:
-      return conf.get(
-          IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
-          IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT);
-    case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
-      return conf.get(
-          IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
-          IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
-    case ErasureCodeConstants.XOR_CODEC_NAME:
-      return conf.get(
-          IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
-          IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
-    default:
-      // For custom codec, we throw exception if the factory is not configured
-      String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
-      String factName = conf.get(rawCoderKey);
-      if (factName == null) {
-        throw new IllegalArgumentException("Raw coder factory not configured " +
-            "for custom codec " + codec);
+  // Return comma separated coder names
+  private static String getRawCoders(Configuration conf, String codec) {
+    return conf.get(
+      IO_ERASURECODE_CODEC + codec + ".rawcoders",
+      DEFAULT_CODERS_MAP.getOrDefault(codec, codec)
+    );
+  }
+
+  private static RawErasureEncoder createRawEncoderWithFallback(
+      Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+    String coders = getRawCoders(conf, codec);
+    for (String factName : Splitter.on(",").split(coders)) {
+      try {
+        if (factName != null) {
+          RawErasureCoderFactory fact = createRawCoderFactory(conf,
+              factName);
+          return fact.createEncoder(coderOptions);
+        }
+      } catch (LinkageError | Exception e) {
+        // Fallback to next coder if possible
+        LOG.warn("Failed to create raw erasure encoder " + factName +
+            ", fallback to next codec if possible", e);
+      }
+    }
+    throw new IllegalArgumentException("Fail to create raw erasure " +
+       "encoder with given codec: " + codec);
+  }
+
+  private static RawErasureDecoder createRawDecoderWithFallback(
+          Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+    String coders = getRawCoders(conf, codec);
+    for (String factName : Splitter.on(",").split(coders)) {
+      try {
+        if (factName != null) {
+          RawErasureCoderFactory fact = createRawCoderFactory(conf,
+              factName);
+          return fact.createDecoder(coderOptions);
+        }
+      } catch (LinkageError | Exception e) {
+        // Fallback to next coder if possible
+        LOG.warn("Failed to create raw erasure decoder " + factName +
+            ", fallback to next codec if possible", e);
       }
-      return factName;
     }
+    throw new IllegalArgumentException("Fail to create raw erasure " +
+            "encoder with given codec: " + codec);
   }
 
   private static ErasureCodec createCodec(Configuration conf,

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -657,34 +657,33 @@
 </property>
 
 <property>
-  <name>io.erasurecode.codec.rs.rawcoder</name>
-  <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
+  <name>io.erasurecode.codec.rs.rawcoders</name>
+  <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
   <description>
-    Raw coder implementation for the rs codec. The default value is a
-    pure Java implementation. There is also a native implementation. Its value
-    is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory.
+    Comma separated raw coder implementations for the rs codec. The earlier
+    factory is prior to followings in case of failure of creating raw coders.
   </description>
 </property>
 
 <property>
-  <name>io.erasurecode.codec.rs-legacy.rawcoder</name>
+  <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
   <value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy</value>
   <description>
-    Raw coder implementation for the rs-legacy codec.
+    Comma separated raw coder implementations for the rs-legacy codec. The earlier
+    factory is prior to followings in case of failure of creating raw coders.
   </description>
 </property>
 
 <property>
-  <name>io.erasurecode.codec.xor.rawcoder</name>
-  <value>org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
+  <name>io.erasurecode.codec.xor.rawcoders</name>
+  <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
   <description>
-    Raw coder implementation for the xor codec. The default value is a pure Java
-    implementation. There is also a native implementation. Its value is
-    org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory.
+    Comma separated raw coder implementations for the xor codec. The earlier
+    factory is prior to followings in case of failure of creating raw coders.
   </description>
 </property>
 
-<!-- file system properties -->
+  <!-- file system properties -->
 
 <property>
   <name>fs.defaultFS</name>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java

@@ -18,12 +18,16 @@
 package org.apache.hadoop.io.erasurecode;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Before;
@@ -70,20 +74,73 @@ public class TestCodecRawCoderMapping {
         numDataUnit, numParityUnit);
 
     String dummyFactName = "DummyNoneExistingFactory";
-    // set the dummy factory to rs-legacy and create a raw coder
-    // with rs, which is OK as the raw coder key is not used
+    // set the dummy factory to raw coders then fail to create any rs raw coder.
     conf.set(CodecUtil.
-        IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
-    RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
-        ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
-    Assert.assertTrue(encoder instanceof RSRawEncoder);
+        IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, dummyFactName);
+    try {
+      CodecUtil.createRawEncoder(conf,
+          ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+      Assert.fail();
+    } catch (Exception e) {
+      GenericTestUtils.assertExceptionContains(
+          "Fail to create raw erasure encoder with given codec: rs", e);
+    }
+
     // now create the raw coder with rs-legacy, which should throw exception
+    conf.set(CodecUtil.
+        IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY, dummyFactName);
     try {
       CodecUtil.createRawEncoder(conf,
           ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
       Assert.fail();
     } catch (Exception e) {
-      GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
+      GenericTestUtils.assertExceptionContains(
+          "Fail to create raw erasure encoder with given codec: rs", e);
     }
   }
+
+  @Test
+  public void testFallbackCoders() {
+    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+            numDataUnit, numParityUnit);
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
+        RSRawErasureCoderFactory.class.getCanonicalName() +
+        "," + NativeRSRawErasureCoderFactory.class.getCanonicalName());
+    // should return default raw coder of rs codec
+    RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+            conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    Assert.assertTrue(encoder instanceof RSRawEncoder);
+    RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+            conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
+    Assert.assertTrue(decoder instanceof RSRawDecoder);
+  }
+
+  @Test
+  public void testLegacyCodecFallback() {
+    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+            numDataUnit, numParityUnit);
+    // should return default raw coder of rs-legacy codec
+    RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+            conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
+    Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
+    RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+            conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
+    Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
+  }
+
+  @Test
+  public void testIgnoreInvalidCodec() {
+    ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+            numDataUnit, numParityUnit);
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
+        "invalid-codec," +
+        "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory");
+    // should return second coder specified by IO_ERASURECODE_CODEC_CODERS
+    RawErasureEncoder encoder = CodecUtil.createRawEncoder(
+            conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    Assert.assertTrue(encoder instanceof XORRawEncoder);
+    RawErasureDecoder decoder = CodecUtil.createRawDecoder(
+            conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
+    Assert.assertTrue(decoder instanceof XORRawDecoder);
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java

@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
      * This tests if the configuration items work or not.
      */
     Configuration conf = new Configuration();
-    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
         RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java

@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
      * This tests if the configuration items work or not.
      */
     Configuration conf = new Configuration();
-    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+    conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
         RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java

@@ -97,7 +97,7 @@ public class TestDFSStripedInputStream {
         getEcPolicy().getName());
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
-          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
           NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     SimulatedFSDataset.setFactory(conf);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java

@@ -84,7 +84,7 @@ public class TestDFSStripedOutputStream {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
-          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
           NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java

@@ -213,7 +213,7 @@ public class TestDFSStripedOutputStreamWithFailure {
     final int numDNs = dataBlocks + parityBlocks;
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
-          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
           NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java

@@ -99,7 +99,7 @@ public class TestReconstructStripedFile {
         false);
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
-          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
           NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java

@@ -67,7 +67,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
-          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+          CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
           NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);