HADOOP-10428. JavaKeyStoreProvider should accept keystore password via configuration falling back to ENV VAR. (tucu)

Conflicts:
	hadoop-common-project/hadoop-common/CHANGES.txt

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1619513 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -102,6 +102,9 @@ Release 2.6.0 - UNRELEASED
 
     HADOOP-10427. KeyProvider implementations should be thread safe. (tucu)
 
+    HADOOP-10428. JavaKeyStoreProvider should accept keystore password via
+    configuration falling back to ENV VAR. (tucu)
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

hadoop-common-project/hadoop-common/pom.xml

@@ -489,6 +489,7 @@
             <exclude>src/test/resources/test.har/_index</exclude>
             <exclude>src/test/resources/test.har/_masterindex</exclude>
             <exclude>src/test/resources/test.har/part-0</exclude>
+            <exclude>src/test/resources/javakeystoreprovider.password</exclude>
           </excludes>
         </configuration>
       </plugin>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.crypto.key;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -27,10 +28,12 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import javax.crypto.spec.SecretKeySpec;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.net.URI;
+import java.net.URL;
 import java.security.Key;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -52,10 +55,21 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
  * any Hadoop FileSystem using the following name mangling:
  *  jks://hdfs@nn1.example.com/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks
  *  jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks
- *
- * The password for the keystore is taken from the HADOOP_KEYSTORE_PASSWORD
- * environment variable with a default of 'none'.
- *
+ * <p/>
+ * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set,
+ * its value is used as the password for the keystore.
+ * <p/>
+ * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set,
+ * the password for the keystore is read from file specified in the
+ * {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file
+ * is looked up in Hadoop's configuration directory via the classpath.
+ * <p/>
+ * <b>NOTE:</b> Make sure the password in the password file does not have an
+ * ENTER at the end, else it won't be valid for the Java KeyStore.
+ * <p/>
+ * If the environment variable, nor the property are not set, the password used
+ * is 'none'.
+ * <p/>
  * It is expected for encrypted InputFormats and OutputFormats to copy the keys
  * from the original provider into the job's Credentials object, which is
  * accessed via the UserProvider. Therefore, this provider won't be used by
@@ -65,16 +79,20 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 public class JavaKeyStoreProvider extends KeyProvider {
   private static final String KEY_METADATA = "KeyMetadata";
   public static final String SCHEME_NAME = "jceks";
-  public static final String KEYSTORE_PASSWORD_NAME =
+
+  public static final String KEYSTORE_PASSWORD_FILE_KEY =
+      "hadoop.security.keystore.java-keystore-provider.password-file";
+
+  public static final String KEYSTORE_PASSWORD_ENV_VAR =
       "HADOOP_KEYSTORE_PASSWORD";
-  public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+  public static final char[] KEYSTORE_PASSWORD_DEFAULT = "none".toCharArray();
 
   private final URI uri;
   private final Path path;
   private final FileSystem fs;
   private final FsPermission permissions;
   private final KeyStore keyStore;
-  private final char[] password;
+  private char[] password;
   private boolean changed = false;
   private Lock readLock;
   private Lock writeLock;
@@ -85,12 +103,29 @@ public class JavaKeyStoreProvider extends KeyProvider {
     this.uri = uri;
     path = unnestUri(uri);
     fs = path.getFileSystem(conf);
-    // Get the password from the user's environment
-    String pw = System.getenv(KEYSTORE_PASSWORD_NAME);
-    if (pw == null) {
-      pw = KEYSTORE_PASSWORD_DEFAULT;
+    // Get the password file from the conf, if not present from the user's
+    // environment var
+    if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
+      password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
+    }
+    if (password == null) {
+      String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+      if (pwFile != null) {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        URL pwdFile = cl.getResource(pwFile);
+        if (pwdFile != null) {
+          InputStream is = pwdFile.openStream();
+          try {
+            password = IOUtils.toCharArray(is);
+          } finally {
+            is.close();
+          }
+        }
+      }
+    }
+    if (password == null) {
+      password = KEYSTORE_PASSWORD_DEFAULT;
     }
-    password = pw.toCharArray();
     try {
       keyStore = KeyStore.getInstance(SCHEME_NAME);
       if (fs.exists(path)) {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
 import org.junit.Test;
 
 import static org.junit.Assert.assertArrayEquals;
@@ -237,4 +238,48 @@ public class TestKeyProviderFactory {
     FileStatus s = fs.getFileStatus(path);
     assertTrue("Permissions should have been retained from the preexisting keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
   }
+
+  @Test
+  public void testJksProviderPasswordViaConfig() throws Exception {
+    Configuration conf = new Configuration();
+    final String ourUrl =
+        JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+    File file = new File(tmpDir, "test.jks");
+    file.delete();
+    try {
+      conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
+      conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY,
+          "javakeystoreprovider.password");
+      KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+      provider.createKey("key3", new byte[32], KeyProvider.options(conf));
+      provider.flush();
+    } catch (Exception ex) {
+      Assert.fail("could not create keystore with password file");
+    }
+    KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+    Assert.assertNotNull(provider.getCurrentKey("key3"));
+
+    try {
+      conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "bar");
+      KeyProviderFactory.getProviders(conf).get(0);
+      Assert.fail("using non existing password file, it should fail");
+    } catch (IOException ex) {
+      //NOP
+    }
+    try {
+      conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "core-site.xml");
+      KeyProviderFactory.getProviders(conf).get(0);
+      Assert.fail("using different password file, it should fail");
+    } catch (IOException ex) {
+      //NOP
+    }
+    try {
+      conf.unset(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY);
+      KeyProviderFactory.getProviders(conf).get(0);
+      Assert.fail("No password file property, env not set, it should fail");
+    } catch (IOException ex) {
+      //NOP
+    }
+  }
+
 }

hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password

@@ -0,0 +1 @@
+foo