Forráskód Böngészése

MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta.

9uapaw 2 éve
szülő
commit
84081a8cae

+ 3 - 2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java

@@ -71,7 +71,6 @@ class JobSubmitter {
   protected static final Logger LOG =
       LoggerFactory.getLogger(JobSubmitter.class);
   private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
-  private static final int SHUFFLE_KEY_LENGTH = 64;
   private FileSystem jtFs;
   private ClientProtocol submitClient;
   private String submitHostName;
@@ -177,7 +176,9 @@ class JobSubmitter {
         KeyGenerator keyGen;
         try {
           keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM);
-          keyGen.init(SHUFFLE_KEY_LENGTH);
+          int shuffleKeyLength =
+              conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH);
+          keyGen.init(shuffleKeyLength);
         } catch (NoSuchAlgorithmException e) {
           throw new IOException("Error generating shuffle secret key", e);
         }

+ 2 - 0
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -1302,4 +1302,6 @@ public interface MRJobConfig {
    */
   @Unstable
   String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must.";
+  String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length";
+  int DEFAULT_SHUFFLE_KEY_LENGTH = 64;
 }