HADOOP-17953. S3A: Tests to lookup global or per-bucket configuration for encryption algorithm (#3525)

Followup to S3-CSE work of HADOOP-13887

Contributed by Mehakmeet Singh

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java

@@ -178,7 +178,8 @@ public abstract class FCStatisticsBaseTest {
    * 
    * @param stats
    */
-  protected abstract void verifyWrittenBytes(Statistics stats);
+  protected abstract void verifyWrittenBytes(Statistics stats)
+      throws IOException;
   
   /**
    * Returns the filesystem uri. Should be set

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java

@@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.file.AccessDeniedException;
 
@@ -41,7 +42,6 @@ import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
 import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
 import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.S3ClientFactory;
@@ -69,6 +69,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
 import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH;
@@ -145,9 +146,14 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
     // disable if assume role opts are off
     assumeSessionTestsEnabled(conf);
     disableFilesystemCaching(conf);
-    String s3EncryptionMethod =
-        conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM,
-            S3AEncryptionMethods.SSE_KMS.getMethod());
+    String s3EncryptionMethod;
+    try {
+      s3EncryptionMethod =
+          getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod();
+    } catch (IOException e) {
+      throw new UncheckedIOException("Failed to lookup encryption algorithm.",
+          e);
+    }
     String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf);
     removeBaseAndBucketOverrides(conf,
         DELEGATION_TOKEN_BINDING,

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java

@@ -13,6 +13,7 @@
  */
 package org.apache.hadoop.fs.s3a.fileContext;
 
+import java.io.IOException;
 import java.net.URI;
 
 import com.amazonaws.services.s3.model.CryptoStorageMode;
@@ -32,9 +33,10 @@ import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
 
 /**
@@ -83,12 +85,14 @@ public class ITestS3AFileContextStatistics extends FCStatisticsBaseTest {
    * @param stats Filesystem statistics.
    */
   @Override
-  protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+  protected void verifyWrittenBytes(FileSystem.Statistics stats)
+      throws IOException {
     //No extra bytes are written
     long expectedBlockSize = blockSize;
-    if (conf.get(S3_ENCRYPTION_ALGORITHM, "")
-        .equals(S3AEncryptionMethods.CSE_KMS.getMethod())) {
-      String keyId = conf.get(S3_ENCRYPTION_KEY, "");
+    if (S3AEncryptionMethods.CSE_KMS.getMethod()
+        .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf)
+            .getMethod())) {
+      String keyId = getS3EncryptionKey(getTestBucketName(conf), conf);
       // Adding padding length and KMS key generation bytes written.
       expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length +
           KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;