HADOOP-14735. ITestS3AEncryptionSSEC failing in parallel s3guard runs.

Contributed by Steve Loughran.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java

@@ -112,10 +112,21 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
    */
   protected Path writeThenReadFile(String name, int len) throws IOException {
     Path path = path(name);
+    writeThenReadFile(path, len);
+    return path;
+  }
+
+  /**
+   * Write a file, read it back, validate the dataset. Overwrites the file
+   * if it is present
+   * @param path path to file
+   * @param len length of file
+   * @throws IOException any IO problem
+   */
+  protected void writeThenReadFile(Path path, int len) throws IOException {
     byte[] data = dataset(len, 'a', 'z');
     writeDataset(getFileSystem(), path, data, data.length, 1024 * 1024, true);
     ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
-    return path;
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java

@@ -18,20 +18,21 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeS3GuardState;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
 import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+
+import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Concrete class that extends {@link AbstractTestS3AEncryption}
@@ -39,17 +40,39 @@ import org.junit.Test;
  */
 public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
 
+  private static final String SERVICE_AMAZON_S3_STATUS_CODE_403
+      = "Service: Amazon S3; Status Code: 403;";
+  private static final String KEY_1
+      = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=";
+  private static final String KEY_2
+      = "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo=";
+  private static final String KEY_3
+      = "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U=";
+  private static final String KEY_4
+      = "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=";
+  private static final int TEST_FILE_LEN = 2048;
+
+  /**
+   * Filesystem created with a different key.
+   */
+  private FileSystem fsKeyB;
+
   @Override
   protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
-    S3ATestUtils.disableFilesystemCaching(conf);
+    disableFilesystemCaching(conf);
     conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
         getSSEAlgorithm().getMethod());
-    conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
-        "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=");
+    conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1);
     return conf;
   }
 
+  @Override
+  public void teardown() throws Exception {
+    super.teardown();
+    IOUtils.closeStream(fsKeyB);
+  }
+
   /**
    * This will create and write to a file using encryption key A, then attempt
    * to read from it again with encryption key B.  This will not work as it
@@ -65,26 +88,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     assumeEnabled();
     skipIfEncryptionTestsDisabled(getConfiguration());
 
-    final Path[] path = new Path[1];
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-
-        int len = 2048;
-        describe("Create an encrypted file of size " + len);
-        String src = createFilename(len);
-        path[0] = writeThenReadFile(src, len);
-
-        //extract the test FS
-        FileSystem fileSystem = createNewFileSystemWithSSECKey(
-            "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s=");
-        byte[] data = dataset(len, 'a', 'z');
-        ContractTestUtils.verifyFileContents(fileSystem, path[0], data);
-        throw new Exception("Fail");
-      });
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          int len = TEST_FILE_LEN;
+          describe("Create an encrypted file of size " + len);
+          Path src = path("testCreateFileAndReadWithDifferentEncryptionKey");
+          writeThenReadFile(src, len);
+
+          //extract the test FS
+          fsKeyB = createNewFileSystemWithSSECKey(
+              "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s=");
+          byte[] data = dataset(len, 'a', 'z');
+          ContractTestUtils.verifyFileContents(fsKeyB, src, data);
+          return fsKeyB.getFileStatus(src);
+        });
   }
 
   /**
-   * While each object has it's own key and should be distinct, this verifies
+   * While each object has its own key and should be distinct, this verifies
    * that hadoop treats object keys as a filesystem path.  So if a top level
    * dir is encrypted with keyA, a sublevel dir cannot be accessed with a
    * different keyB.
@@ -99,24 +121,18 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     skipIfEncryptionTestsDisabled(getConfiguration());
     assumeS3GuardState(false, getConfiguration());
 
-    final Path[] path = new Path[1];
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-
-        path[0] = S3ATestUtils.createTestPath(
-          new Path(createFilename("dir/"))
-        );
-        Path nestedDirectory = S3ATestUtils.createTestPath(
-            new Path(createFilename("dir/nestedDir/"))
-        );
-        FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-            "G61nz31Q7+zpjJWbakxfTOZW4VS0UmQWAq2YXhcTXoo=");
-        getFileSystem().mkdirs(path[0]);
-        fsKeyB.mkdirs(nestedDirectory);
-
-        throw new Exception("Exception should be thrown.");
-      });
-    rm(getFileSystem(), path[0], true, false);
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          Path base = path("testCreateSubdirWithDifferentKey");
+          Path nestedDirectory = new Path(base, "nestedDir");
+          fsKeyB = createNewFileSystemWithSSECKey(
+              KEY_2);
+          getFileSystem().mkdirs(base);
+          fsKeyB.mkdirs(nestedDirectory);
+          // expected to fail
+          return fsKeyB.getFileStatus(nestedDirectory);
+        });
   }
 
   /**
@@ -132,20 +148,17 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     assumeEnabled();
     skipIfEncryptionTestsDisabled(getConfiguration());
 
-    final Path[] path = new Path[1];
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-
-        int len = 2048;
-        String src = createFilename(len);
-        path[0] = writeThenReadFile(src, len);
-
-        FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-            "NTx0dUPrxoo9+LbNiT/gqf3z9jILqL6ilismFmJO50U=");
-        fsKeyB.rename(path[0], new Path(createFilename("different-path.txt")));
-
-        throw new Exception("Exception should be thrown.");
-      });
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          int len = TEST_FILE_LEN;
+          Path src = path(createFilename(len));
+          writeThenReadFile(src, len);
+          fsKeyB = createNewFileSystemWithSSECKey(KEY_3);
+          Path dest = path(createFilename("different-path.txt"));
+          getFileSystem().mkdirs(dest.getParent());
+          return fsKeyB.rename(src, dest);
+        });
   }
 
   /**
@@ -159,11 +172,11 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     assumeEnabled();
     skipIfEncryptionTestsDisabled(getConfiguration());
 
-    String src = createFilename("original-path.txt");
-    Path path = writeThenReadFile(src, 2048);
-    Path newPath = path(createFilename("different-path.txt"));
-    getFileSystem().rename(path, newPath);
-    byte[] data = dataset(2048, 'a', 'z');
+    Path src = path("original-path.txt");
+    writeThenReadFile(src, TEST_FILE_LEN);
+    Path newPath = path("different-path.txt");
+    getFileSystem().rename(src, newPath);
+    byte[] data = dataset(TEST_FILE_LEN, 'a', 'z');
     ContractTestUtils.verifyFileContents(getFileSystem(), newPath, data);
   }
 
@@ -179,29 +192,24 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     skipIfEncryptionTestsDisabled(getConfiguration());
     assumeS3GuardState(false, getConfiguration());
 
-    Path nestedDirectory = S3ATestUtils.createTestPath(
-         path(createFilename("/a/b/c/"))
-    );
+    Path pathABC = path("testListEncryptedDir/a/b/c/");
+    Path pathAB = pathABC.getParent();
+    Path pathA = pathAB.getParent();
+
+    Path nestedDirectory = createTestPath(pathABC);
     assertTrue(getFileSystem().mkdirs(nestedDirectory));
 
-    FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-        "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+    fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
 
-    fsKeyB.listFiles(S3ATestUtils.createTestPath(
-        path(createFilename("/a/"))
-    ), true);
-    fsKeyB.listFiles(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/"))
-    ), true);
+    fsKeyB.listFiles(pathA, true);
+    fsKeyB.listFiles(pathAB, true);
 
     //Until this point, no exception is thrown about access
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-        fsKeyB.listFiles(S3ATestUtils.createTestPath(
-            path(createFilename("/a/b/c/"))
-        ), false);
-        throw new Exception("Exception should be thrown.");
-      });
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          fsKeyB.listFiles(pathABC, false);
+        });
 
     Configuration conf = this.createConfiguration();
     conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM);
@@ -212,22 +220,13 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     FileSystem unencryptedFileSystem = contract.getTestFileSystem();
 
     //unencrypted can access until the final directory
-    unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
-        path(createFilename("/a/"))
-    ), true);
-    unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/"))
-    ), true);
-    intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class,
-        "Bad Request (Service: Amazon S3; Status Code: 400; Error" +
-          " Code: 400 Bad Request;", () -> {
-
-        unencryptedFileSystem.listFiles(S3ATestUtils.createTestPath(
-            path(createFilename("/a/b/c/"))
-        ), false);
-        throw new Exception("Exception should be thrown.");
-      });
-    rm(getFileSystem(), path(createFilename("/")), true, false);
+    unencryptedFileSystem.listFiles(pathA, true);
+    unencryptedFileSystem.listFiles(pathAB, true);
+    AWSS3IOException ex = intercept(AWSS3IOException.class,
+        () -> {
+          unencryptedFileSystem.listFiles(pathABC, false);
+        });
+    assertStatusCode(ex, 400);
   }
 
   /**
@@ -241,30 +240,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     skipIfEncryptionTestsDisabled(getConfiguration());
     assumeS3GuardState(false, getConfiguration());
 
-    Path nestedDirectory = S3ATestUtils.createTestPath(
-         path(createFilename("/a/b/c/"))
-    );
-    assertTrue(getFileSystem().mkdirs(nestedDirectory));
+    Path pathABC = path("testListStatusEncryptedDir/a/b/c/");
+    Path pathAB = pathABC.getParent();
+    Path pathA = pathAB.getParent();
+    assertTrue(getFileSystem().mkdirs(pathABC));
 
-    FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-        "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+    fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
 
-    fsKeyB.listStatus(S3ATestUtils.createTestPath(
-        path(createFilename("/a/"))));
-    fsKeyB.listStatus(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/"))));
+    fsKeyB.listStatus(pathA);
+    fsKeyB.listStatus(pathAB);
 
     //Until this point, no exception is thrown about access
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-        fsKeyB.listStatus(S3ATestUtils.createTestPath(
-            path(createFilename("/a/b/c/"))));
-
-        throw new Exception("Exception should be thrown.");
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          fsKeyB.listStatus(pathABC);
         });
 
     //Now try it with an unencrypted filesystem.
-    Configuration conf = this.createConfiguration();
+    Configuration conf = createConfiguration();
     conf.unset(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM);
     conf.unset(Constants.SERVER_SIDE_ENCRYPTION_KEY);
 
@@ -273,20 +267,14 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     FileSystem unencryptedFileSystem = contract.getTestFileSystem();
 
     //unencrypted can access until the final directory
-    unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
-        path(createFilename("/a/"))));
-    unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/"))));
-
-    intercept(org.apache.hadoop.fs.s3a.AWSS3IOException.class,
-        "Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400" +
-        " Bad Request;", () -> {
-
-        unencryptedFileSystem.listStatus(S3ATestUtils.createTestPath(
-            path(createFilename("/a/b/c/"))));
-        throw new Exception("Exception should be thrown.");
+    unencryptedFileSystem.listStatus(pathA);
+    unencryptedFileSystem.listStatus(pathAB);
+
+    AWSS3IOException ex = intercept(AWSS3IOException.class,
+        () -> {
+          unencryptedFileSystem.listStatus(pathABC);
         });
-    rm(getFileSystem(), path(createFilename("/")), true, false);
+    assertStatusCode(ex, 400);
   }
 
   /**
@@ -299,31 +287,23 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     assumeEnabled();
     skipIfEncryptionTestsDisabled(getConfiguration());
     assumeS3GuardState(false, getConfiguration());
+    Path pathABC = path("testListStatusEncryptedFile/a/b/c/");
+    assertTrue(getFileSystem().mkdirs(pathABC));
 
-    Path nestedDirectory = S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/c/"))
-    );
-    assertTrue(getFileSystem().mkdirs(nestedDirectory));
-
-    String src = createFilename("/a/b/c/fileToStat.txt");
-    Path fileToStat =  writeThenReadFile(src, 2048);
+    Path fileToStat = new Path(pathABC, "fileToStat.txt");
+    writeThenReadFile(fileToStat, TEST_FILE_LEN);
 
-    FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-        "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
+    fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
 
     //Until this point, no exception is thrown about access
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Service: Amazon S3; Status Code: 403;", () -> {
-        fsKeyB.listStatus(S3ATestUtils.createTestPath(fileToStat));
-
-        throw new Exception("Exception should be thrown.");
-      });
-    rm(getFileSystem(), path(createFilename("/")), true, false);
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          fsKeyB.listStatus(fileToStat);
+        });
   }
 
 
-
-
   /**
    * It is possible to delete directories without the proper encryption key and
    * the hierarchy above it.
@@ -335,31 +315,25 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
     assumeEnabled();
     skipIfEncryptionTestsDisabled(getConfiguration());
     assumeS3GuardState(false, getConfiguration());
-
-    Path nestedDirectory = S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/c/"))
-    );
-    assertTrue(getFileSystem().mkdirs(nestedDirectory));
-    String src = createFilename("/a/b/c/filetobedeleted.txt");
-    Path fileToDelete =  writeThenReadFile(src, 2048);
-
-    FileSystem fsKeyB = createNewFileSystemWithSSECKey(
-        "msdo3VvvZznp66Gth58a91Hxe/UpExMkwU9BHkIjfW8=");
-    intercept(java.nio.file.AccessDeniedException.class,
-        "Forbidden (Service: Amazon S3; Status Code: 403; Error Code: " +
-        "403 Forbidden", () -> {
-
-        fsKeyB.delete(fileToDelete, false);
-        throw new Exception("Exception should be thrown.");
-      });
+    Path pathABC = path("testDeleteEncryptedObjectWithDifferentKey/a/b/c/");
+
+    Path pathAB = pathABC.getParent();
+    Path pathA = pathAB.getParent();
+    assertTrue(getFileSystem().mkdirs(pathABC));
+    Path fileToDelete = new Path(pathABC, "filetobedeleted.txt");
+    writeThenReadFile(fileToDelete, TEST_FILE_LEN);
+    fsKeyB = createNewFileSystemWithSSECKey(KEY_4);
+    intercept(AccessDeniedException.class,
+        SERVICE_AMAZON_S3_STATUS_CODE_403,
+        () -> {
+          fsKeyB.delete(fileToDelete, false);
+        });
 
     //This is possible
-    fsKeyB.delete(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/c/"))), true);
-    fsKeyB.delete(S3ATestUtils.createTestPath(
-        path(createFilename("/a/b/"))), true);
-    fsKeyB.delete(S3ATestUtils.createTestPath(
-        path(createFilename("/a/"))), true);
+    fsKeyB.delete(pathABC, true);
+    fsKeyB.delete(pathAB, true);
+    fsKeyB.delete(pathA, true);
+    assertPathDoesNotExist("expected recursive delete", fileToDelete);
   }
 
   private FileSystem createNewFileSystemWithSSECKey(String sseCKey) throws
@@ -377,4 +351,5 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
   protected S3AEncryptionMethods getSSEAlgorithm() {
     return S3AEncryptionMethods.SSE_C;
   }
+
 }