HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for generation/decryption of keys. (tucu)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -476,6 +476,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
     6.x version. (rkanter via tucu)
 
+    HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for 
+    generation/decryption of keys. (tucu)
+
   OPTIMIZATIONS
 
     HADOOP-10838. Byte array native checksumming. (James Thomas via todd)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java

@@ -108,6 +108,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
   private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
 
   private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
+    super(conf);
     this.uri = uri;
     path = ProviderUtils.unnestUri(uri);
     fs = path.getFileSystem(conf);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java

@@ -56,6 +56,8 @@ public abstract class KeyProvider {
       "hadoop.security.key.default.bitlength";
   public static final int DEFAULT_BITLENGTH = 128;
 
+  private final Configuration conf;
+
   /**
    * The combination of both the key version name and the key material.
    */
@@ -353,6 +355,24 @@ public abstract class KeyProvider {
     }
   }
 
+  /**
+   * Constructor.
+   * 
+   * @param conf configuration for the provider
+   */
+  public KeyProvider(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  /**
+   * Return the provider configuration.
+   * 
+   * @return the provider configuration
+   */
+  public Configuration getConf() {
+    return conf;
+  }
+  
   /**
    * A helper function to create an options object.
    * @param conf the configuration to use

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.crypto.key;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 
@@ -29,6 +30,9 @@ import javax.crypto.spec.SecretKeySpec;
 import com.google.common.base.Preconditions;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.Decryptor;
+import org.apache.hadoop.crypto.Encryptor;
 
 /**
  * A KeyProvider with Cryptographic Extensions specifically for generating
@@ -239,18 +243,25 @@ public class KeyProviderCryptoExtension extends
       Preconditions.checkNotNull(encryptionKey,
           "No KeyVersion exists for key '%s' ", encryptionKeyName);
       // Generate random bytes for new key and IV
-      Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
+
+      CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
       final byte[] newKey = new byte[encryptionKey.getMaterial().length];
-      RANDOM.get().nextBytes(newKey);
-      final byte[] iv = new byte[cipher.getBlockSize()];
-      RANDOM.get().nextBytes(iv);
+      cc.generateSecureRandom(newKey);
+      final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
+      cc.generateSecureRandom(iv);
       // Encryption key IV is derived from new key's IV
       final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
-      // Encrypt the new key
-      cipher.init(Cipher.ENCRYPT_MODE,
-          new SecretKeySpec(encryptionKey.getMaterial(), "AES"),
-          new IvParameterSpec(encryptionIV));
-      final byte[] encryptedKey = cipher.doFinal(newKey);
+      Encryptor encryptor = cc.createEncryptor();
+      encryptor.init(encryptionKey.getMaterial(), encryptionIV);
+      int keyLen = newKey.length;
+      ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+      ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+      bbIn.put(newKey);
+      bbIn.flip();
+      encryptor.encrypt(bbIn, bbOut);
+      bbOut.flip();
+      byte[] encryptedKey = new byte[keyLen];
+      bbOut.get(encryptedKey);    
       return new EncryptedKeyVersion(encryptionKeyName,
           encryptionKey.getVersionName(), iv,
           new KeyVersion(encryptionKey.getName(), EEK, encryptedKey));
@@ -274,19 +285,25 @@ public class KeyProviderCryptoExtension extends
                 KeyProviderCryptoExtension.EEK,
                 encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
             );
-      final byte[] encryptionKeyMaterial = encryptionKey.getMaterial();
+
       // Encryption key IV is determined from encrypted key's IV
       final byte[] encryptionIV =
           EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());
-      // Init the cipher with encryption key parameters
-      Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
-      cipher.init(Cipher.DECRYPT_MODE,
-          new SecretKeySpec(encryptionKeyMaterial, "AES"),
-          new IvParameterSpec(encryptionIV));
-      // Decrypt the encrypted key
+
+      CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
+      Decryptor decryptor = cc.createDecryptor();
+      decryptor.init(encryptionKey.getMaterial(), encryptionIV);
       final KeyVersion encryptedKV =
           encryptedKeyVersion.getEncryptedKeyVersion();
-      final byte[] decryptedKey = cipher.doFinal(encryptedKV.getMaterial());
+      int keyLen = encryptedKV.getMaterial().length;
+      ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+      ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+      bbIn.put(encryptedKV.getMaterial());
+      bbIn.flip();
+      decryptor.decrypt(bbIn, bbOut);
+      bbOut.flip();
+      byte[] decryptedKey = new byte[keyLen];
+      bbOut.get(decryptedKey);
       return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
     }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java

@@ -40,6 +40,7 @@ public abstract class KeyProviderExtension
   private E extension;
 
   public KeyProviderExtension(KeyProvider keyProvider, E extensions) {
+    super(keyProvider.getConf());
     this.keyProvider = keyProvider;
     this.extension = extensions;
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java

@@ -44,7 +44,8 @@ public class UserProvider extends KeyProvider {
   private final Credentials credentials;
   private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
 
-  private UserProvider() throws IOException {
+  private UserProvider(Configuration conf) throws IOException {
+    super(conf);
     user = UserGroupInformation.getCurrentUser();
     credentials = user.getCredentials();
   }
@@ -145,7 +146,7 @@ public class UserProvider extends KeyProvider {
     public KeyProvider createProvider(URI providerName,
                                       Configuration conf) throws IOException {
       if (SCHEME_NAME.equals(providerName.getScheme())) {
-        return new UserProvider();
+        return new UserProvider(conf);
       }
       return null;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java

@@ -283,6 +283,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   }
 
   public KMSClientProvider(URI uri, Configuration conf) throws IOException {
+    super(conf);
     Path path = ProviderUtils.unnestUri(uri);
     URL url = path.toUri().toURL();
     kmsUrl = createServiceURL(url);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.crypto.key;
 
 import java.util.Date;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
 import org.junit.Assert;
 import org.junit.Test;
@@ -32,6 +33,7 @@ public class TestCachingKeyProvider {
     KeyProvider mockProv = Mockito.mock(KeyProvider.class);
     Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
     Mockito.when(mockProv.getCurrentKey(Mockito.eq("k2"))).thenReturn(null);
+    Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
     KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
 
     // asserting caching
@@ -58,6 +60,7 @@ public class TestCachingKeyProvider {
     Mockito.when(mockProv.getKeyVersion(Mockito.eq("k1@0")))
         .thenReturn(mockKey);
     Mockito.when(mockProv.getKeyVersion(Mockito.eq("k2@0"))).thenReturn(null);
+    Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
     KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
 
     // asserting caching
@@ -88,6 +91,7 @@ public class TestCachingKeyProvider {
     KeyProvider mockProv = Mockito.mock(KeyProvider.class);
     Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(mockMeta);
     Mockito.when(mockProv.getMetadata(Mockito.eq("k2"))).thenReturn(null);
+    Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
     KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
 
     // asserting caching
@@ -112,6 +116,7 @@ public class TestCachingKeyProvider {
     KeyProvider.KeyVersion mockKey = Mockito.mock(KeyProvider.KeyVersion.class);
     KeyProvider mockProv = Mockito.mock(KeyProvider.class);
     Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
+    Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
     KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
     Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
     Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));
@@ -134,6 +139,7 @@ public class TestCachingKeyProvider {
         .thenReturn(mockKey);
     Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(
         new KMSClientProvider.KMSMetadata("c", 0, "l", null, new Date(), 1));
+    Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
     KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
     Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
     Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java

@@ -159,6 +159,10 @@ public class TestKeyProvider {
     private int size;
     private byte[] material;
 
+    public MyKeyProvider(Configuration conf) {
+      super(conf);
+    }
+
     @Override
     public KeyVersion getKeyVersion(String versionName)
         throws IOException {
@@ -216,7 +220,7 @@ public class TestKeyProvider {
 
   @Test
   public void testMaterialGeneration() throws Exception {
-    MyKeyProvider kp = new MyKeyProvider();
+    MyKeyProvider kp = new MyKeyProvider(new Configuration());
     KeyProvider.Options options = new KeyProvider.Options(new Configuration());
     options.setCipher(CIPHER);
     options.setBitLength(128);
@@ -225,10 +229,19 @@ public class TestKeyProvider {
     Assert.assertEquals(CIPHER, kp.algorithm);
     Assert.assertNotNull(kp.material);
 
-    kp = new MyKeyProvider();
+    kp = new MyKeyProvider(new Configuration());
     kp.rollNewVersion("hello");
     Assert.assertEquals(128, kp.size);
     Assert.assertEquals(CIPHER, kp.algorithm);
     Assert.assertNotNull(kp.material);
   }
+
+  @Test
+  public void testConfiguration() throws Exception {
+    Configuration conf = new Configuration(false);
+    conf.set("a", "A");
+    MyKeyProvider kp = new MyKeyProvider(conf);
+    Assert.assertEquals("A", kp.getConf().get("a"));
+  }
+
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java

@@ -29,13 +29,18 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 public class TestKeyProviderDelegationTokenExtension {
   
   public static abstract class MockKeyProvider extends
       KeyProvider implements DelegationTokenExtension {
+
+    public MockKeyProvider() {
+      super(new Configuration(false));
+    }
   }
-  
+
   @Test
   public void testCreateExtension() throws Exception {
     Configuration conf = new Configuration();
@@ -50,9 +55,11 @@ public class TestKeyProviderDelegationTokenExtension {
     Assert.assertNull(kpDTE1.addDelegationTokens("user", credentials));
     
     MockKeyProvider mock = mock(MockKeyProvider.class);
+    Mockito.when(mock.getConf()).thenReturn(new Configuration());
     when(mock.addDelegationTokens("renewer", credentials)).thenReturn(
-        new Token<?>[] { new Token(null, null, new Text("kind"), new Text(
-            "service")) });
+        new Token<?>[]{new Token(null, null, new Text("kind"), new Text(
+            "service"))}
+    );
     KeyProviderDelegationTokenExtension kpDTE2 =
         KeyProviderDelegationTokenExtension
         .createKeyProviderDelegationTokenExtension(mock);