HADOOP-14777. S3Guard premerge changes: java 7 build & test tuning

Contributed by Steve Loughran.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java

@@ -751,7 +751,7 @@ public abstract class FileSystemContractBaseTest {
    * renames empty dirs doesn't cause any regressions.
    */
   public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable {
-    if (!renameSupported()) return;
+    assumeTrue(renameSupported());
     Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed");
     fs.mkdirs(parentdir);
     Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest");

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java

@@ -168,9 +168,8 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
 
   @Override
   public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
-      deleteObjectsRequest) throws AmazonClientException,
-      AmazonServiceException
-  {
+      deleteObjectsRequest)
+      throws AmazonClientException, AmazonServiceException {
     for (DeleteObjectsRequest.KeyVersion keyVersion :
         deleteObjectsRequest.getKeys()) {
       registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest
@@ -404,8 +403,9 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
     private final List<S3ObjectSummary> customListing;
     private final List<String> customPrefixes;
 
-    public CustomObjectListing(ObjectListing rawListing,
-        List<S3ObjectSummary> customListing, List<String> customPrefixes) {
+    CustomObjectListing(ObjectListing rawListing,
+        List<S3ObjectSummary> customListing,
+        List<String> customPrefixes) {
       super();
       this.customListing = customListing;
       this.customPrefixes = customPrefixes;

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java

@@ -345,7 +345,7 @@ public class S3AInstrumentation {
   public MutableQuantiles lookupQuantiles(String name) {
     MutableMetric metric = lookupMetric(name);
     if (metric == null) {
-        LOG.debug("No quantiles {}", name);
+      LOG.debug("No quantiles {}", name);
     }
     return (MutableQuantiles) metric;
   }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -222,11 +222,7 @@ public class DirListingMetadata {
     Path childPath = childStatusToPathKey(childFileStatus);
     PathMetadata newValue = new PathMetadata(childFileStatus);
     PathMetadata oldValue = listMap.put(childPath, newValue);
-    if (oldValue == null) {
-      return true;
-    } else {
-      return !oldValue.equals(newValue);
-    }
+    return oldValue == null || !oldValue.equals(newValue);
   }
 
   @Override

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java

@@ -115,12 +115,12 @@ public interface DynamoDBClientFactory extends Configurable {
     }
 
     private static String validRegionsString() {
-      final String DELIMITER = ", ";
+      final String delimiter = ", ";
       Regions[] regions = Regions.values();
       StringBuilder sb = new StringBuilder();
       for (int i = 0; i < regions.length; i++) {
         if (i > 0) {
-          sb.append(DELIMITER);
+          sb.append(delimiter);
         }
         sb.append(regions[i].getName());
       }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java

@@ -481,8 +481,8 @@ public class DynamoDBMetadataStore implements MetadataStore {
         parent = parent.getParent();
       }
     }
-     return ancestry.values();
-   }
+    return ancestry.values();
+  }
 
   @Override
   public void move(Collection<Path> pathsToDelete,
@@ -893,10 +893,13 @@ public class DynamoDBMetadataStore implements MetadataStore {
 
   /**
    * Wait for table being active.
+   * @param t table to block on.
+   * @throws IOException IO problems
+   * @throws InterruptedIOException if the wait was interrupted
    */
-  private void waitForTableActive(Table table) throws IOException {
+  private void waitForTableActive(Table t) throws IOException {
     try {
-      table.waitForActive();
+      t.waitForActive();
     } catch (InterruptedException e) {
       LOG.warn("Interrupted while waiting for table {} in region {} active",
           tableName, region, e);
@@ -910,7 +913,8 @@ public class DynamoDBMetadataStore implements MetadataStore {
    * Create a table, wait for it to become active, then add the version
    * marker.
    * @param capacity capacity to provision
-   * @throws IOException on an failure.
+   * @throws IOException on any failure.
+   * @throws InterruptedIOException if the wait was interrupted
    */
   private void createTable(ProvisionedThroughput capacity) throws IOException {
     try {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java

@@ -34,7 +34,6 @@ public class NullMetadataStore implements MetadataStore {
 
   @Override
   public void initialize(FileSystem fs) throws IOException {
-    return;
   }
 
   @Override

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java

@@ -310,7 +310,6 @@ public final class S3Guard {
       ms.put(pathMetas);
     } catch (IOException ioe) {
       LOG.error("MetadataStore#put() failure:", ioe);
-      return;
     }
   }
 
@@ -429,8 +428,9 @@ public final class S3Guard {
   }
 
   private static void addMoveStatus(Collection<Path> srcPaths,
-      Collection<PathMetadata> dstMetas, Path srcPath, FileStatus dstStatus)
-  {
+      Collection<PathMetadata> dstMetas,
+      Path srcPath,
+      FileStatus dstStatus) {
     srcPaths.add(srcPath);
     dstMetas.add(new PathMetadata(dstStatus));
   }
@@ -446,7 +446,7 @@ public final class S3Guard {
     // multiple S3AFileSystem instances
     Preconditions.checkNotNull(uri.getHost(), "Null host in " + uri);
 
-    // I believe this should never fail, since there is a host?
+    // This should never fail, but is retained for completeness.
     Preconditions.checkNotNull(uri.getScheme(), "Null scheme in " + uri);
   }
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -85,9 +86,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
   static final int INVALID_ARGUMENT = 1;
   static final int ERROR = 99;
 
-  protected S3AFileSystem s3a;
-  protected MetadataStore ms;
-  protected CommandFormat commandFormat;
+  private S3AFileSystem filesystem;
+  private MetadataStore store;
+  private final CommandFormat commandFormat;
 
   private static final String META_FLAG = "meta";
   private static final String DAYS_FLAG = "days";
@@ -118,11 +119,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
    */
   abstract String getName();
 
-  @VisibleForTesting
-  public MetadataStore getMetadataStore() {
-    return ms;
-  }
-
   /**
    * Parse DynamoDB region from either -m option or a S3 path.
    *
@@ -135,7 +131,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
    */
   boolean parseDynamoDBRegion(List<String> paths) throws IOException {
     Configuration conf = getConf();
-    String fromCli = commandFormat.getOptValue(REGION_FLAG);
+    String fromCli = getCommandFormat().getOptValue(REGION_FLAG);
     String fromConf = conf.get(S3GUARD_DDB_REGION_KEY);
     boolean hasS3Path = !paths.isEmpty();
 
@@ -181,26 +177,26 @@ public abstract class S3GuardTool extends Configured implements Tool {
    * @return a initialized metadata store.
    */
   MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
-    if (ms != null) {
-      return ms;
+    if (getStore() != null) {
+      return getStore();
     }
     Configuration conf;
-    if (s3a == null) {
+    if (filesystem == null) {
       conf = getConf();
     } else {
-      conf = s3a.getConf();
+      conf = filesystem.getConf();
     }
-    String metaURI = commandFormat.getOptValue(META_FLAG);
+    String metaURI = getCommandFormat().getOptValue(META_FLAG);
     if (metaURI != null && !metaURI.isEmpty()) {
       URI uri = URI.create(metaURI);
       LOG.info("create metadata store: {}", uri + " scheme: "
           + uri.getScheme());
-      switch (uri.getScheme().toLowerCase()) {
+      switch (uri.getScheme().toLowerCase(Locale.ENGLISH)) {
       case "local":
-        ms = new LocalMetadataStore();
+        setStore(new LocalMetadataStore());
         break;
       case "dynamodb":
-        ms = new DynamoDBMetadataStore();
+        setStore(new DynamoDBMetadataStore());
         conf.set(S3GUARD_DDB_TABLE_NAME_KEY, uri.getAuthority());
         if (forceCreate) {
           conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
@@ -213,19 +209,19 @@ public abstract class S3GuardTool extends Configured implements Tool {
     } else {
       // CLI does not specify metadata store URI, it uses default metadata store
       // DynamoDB instead.
-      ms = new DynamoDBMetadataStore();
+      setStore(new DynamoDBMetadataStore());
       if (forceCreate) {
         conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
       }
     }
 
-    if (s3a == null) {
-      ms.initialize(conf);
+    if (filesystem == null) {
+      getStore().initialize(conf);
     } else {
-      ms.initialize(s3a);
+      getStore().initialize(filesystem);
     }
-    LOG.info("Metadata store {} is initialized.", ms);
-    return ms;
+    LOG.info("Metadata store {} is initialized.", getStore());
+    return getStore();
   }
 
   /**
@@ -252,7 +248,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
           String.format("URI %s is not a S3A file system: %s", uri,
               fs.getClass().getName()));
     }
-    s3a = (S3AFileSystem) fs;
+    filesystem = (S3AFileSystem) fs;
   }
 
   /**
@@ -263,7 +259,30 @@ public abstract class S3GuardTool extends Configured implements Tool {
    * @return the position arguments from CLI.
    */
   List<String> parseArgs(String[] args) {
-    return commandFormat.parse(args, 1);
+    return getCommandFormat().parse(args, 1);
+  }
+
+  protected S3AFileSystem getFilesystem() {
+    return filesystem;
+  }
+
+  protected void setFilesystem(S3AFileSystem filesystem) {
+    this.filesystem = filesystem;
+  }
+
+  @VisibleForTesting
+  public MetadataStore getStore() {
+    return store;
+  }
+
+  @VisibleForTesting
+  protected void setStore(MetadataStore store) {
+    Preconditions.checkNotNull(store);
+    this.store = store;
+  }
+
+  protected CommandFormat getCommandFormat() {
+    return commandFormat;
   }
 
   /**
@@ -290,9 +309,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
     Init(Configuration conf) {
       super(conf);
       // read capacity.
-      commandFormat.addOptionWithValue(READ_FLAG);
+      getCommandFormat().addOptionWithValue(READ_FLAG);
       // write capacity.
-      commandFormat.addOptionWithValue(WRITE_FLAG);
+      getCommandFormat().addOptionWithValue(WRITE_FLAG);
     }
 
     @Override
@@ -309,12 +328,12 @@ public abstract class S3GuardTool extends Configured implements Tool {
     public int run(String[] args) throws IOException {
       List<String> paths = parseArgs(args);
 
-      String readCap = commandFormat.getOptValue(READ_FLAG);
+      String readCap = getCommandFormat().getOptValue(READ_FLAG);
       if (readCap != null && !readCap.isEmpty()) {
         int readCapacity = Integer.parseInt(readCap);
         getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, readCapacity);
       }
-      String writeCap = commandFormat.getOptValue(WRITE_FLAG);
+      String writeCap = getCommandFormat().getOptValue(WRITE_FLAG);
       if (writeCap != null && !writeCap.isEmpty()) {
         int writeCapacity = Integer.parseInt(writeCap);
         getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCapacity);
@@ -380,9 +399,10 @@ public abstract class S3GuardTool extends Configured implements Tool {
         return SUCCESS;
       }
 
-      Preconditions.checkState(ms != null, "Metadata Store is not initialized");
+      Preconditions.checkState(getStore() != null,
+          "Metadata Store is not initialized");
 
-      ms.destroy();
+      getStore().destroy();
       LOG.info("Metadata store is deleted.");
       return SUCCESS;
     }
@@ -414,12 +434,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
       super(conf);
     }
 
-    // temporary: for metadata store.
-    @VisibleForTesting
-    void setMetadataStore(MetadataStore ms) {
-      this.ms = ms;
-    }
-
     @Override
     String getName() {
       return NAME;
@@ -445,7 +459,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
         }
         FileStatus dir = DynamoDBMetadataStore.makeDirStatus(parent,
             f.getOwner());
-        ms.put(new PathMetadata(dir));
+        getStore().put(new PathMetadata(dir));
         dirCache.add(parent);
         parent = parent.getParent();
       }
@@ -458,8 +472,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
      */
     private long importDir(FileStatus status) throws IOException {
       Preconditions.checkArgument(status.isDirectory());
-      RemoteIterator<LocatedFileStatus> it =
-          s3a.listFilesAndEmptyDirectories(status.getPath(), true);
+      RemoteIterator<LocatedFileStatus> it = getFilesystem()
+          .listFilesAndEmptyDirectories(status.getPath(), true);
       long items = 0;
 
       while (it.hasNext()) {
@@ -477,7 +491,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
               located.getOwner());
         }
         putParentsIfNotPresent(child);
-        ms.put(new PathMetadata(child));
+        getStore().put(new PathMetadata(child));
         items++;
       }
       return items;
@@ -506,14 +520,14 @@ public abstract class S3GuardTool extends Configured implements Tool {
         filePath = "/";
       }
       Path path = new Path(filePath);
-      FileStatus status = s3a.getFileStatus(path);
+      FileStatus status = getFilesystem().getFileStatus(path);
 
       initMetadataStore(false);
 
       long items = 1;
       if (status.isFile()) {
         PathMetadata meta = new PathMetadata(status);
-        ms.put(meta);
+        getStore().put(meta);
       } else {
         items = importDir(status);
       }
@@ -552,12 +566,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
       super(conf);
     }
 
-    @VisibleForTesting
-    void setMetadataStore(MetadataStore ms) {
-      Preconditions.checkNotNull(ms);
-      this.ms = ms;
-    }
-
     @Override
     String getName() {
       return NAME;
@@ -651,7 +659,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
 
       Map<Path, FileStatus> s3Children = new HashMap<>();
       if (s3Dir != null && s3Dir.isDirectory()) {
-        for (FileStatus status : s3a.listStatus(s3Dir.getPath())) {
+        for (FileStatus status : getFilesystem().listStatus(s3Dir.getPath())) {
           s3Children.put(status.getPath(), status);
         }
       }
@@ -659,7 +667,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
       Map<Path, FileStatus> msChildren = new HashMap<>();
       if (msDir != null && msDir.isDirectory()) {
         DirListingMetadata dirMeta =
-            ms.listChildren(msDir.getPath());
+            getStore().listChildren(msDir.getPath());
 
         if (dirMeta != null) {
           for (PathMetadata meta : dirMeta.getListing()) {
@@ -692,13 +700,13 @@ public abstract class S3GuardTool extends Configured implements Tool {
      * @throws IOException on I/O errors.
      */
     private void compareRoot(Path path, PrintStream out) throws IOException {
-      Path qualified = s3a.qualify(path);
+      Path qualified = getFilesystem().qualify(path);
       FileStatus s3Status = null;
       try {
-        s3Status = s3a.getFileStatus(qualified);
+        s3Status = getFilesystem().getFileStatus(qualified);
       } catch (FileNotFoundException e) {
       }
-      PathMetadata meta = ms.get(qualified);
+      PathMetadata meta = getStore().get(qualified);
       FileStatus msStatus = (meta != null && !meta.isDeleted()) ?
           meta.getFileStatus() : null;
       compareDir(msStatus, s3Status, out);
@@ -727,7 +735,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
       } else {
         root = new Path(uri.getPath());
       }
-      root = s3a.qualify(root);
+      root = getFilesystem().qualify(root);
       compareRoot(root, out);
       out.flush();
       return SUCCESS;
@@ -763,16 +771,17 @@ public abstract class S3GuardTool extends Configured implements Tool {
     Prune(Configuration conf) {
       super(conf);
 
-      commandFormat.addOptionWithValue(DAYS_FLAG);
-      commandFormat.addOptionWithValue(HOURS_FLAG);
-      commandFormat.addOptionWithValue(MINUTES_FLAG);
-      commandFormat.addOptionWithValue(SECONDS_FLAG);
+      CommandFormat format = getCommandFormat();
+      format.addOptionWithValue(DAYS_FLAG);
+      format.addOptionWithValue(HOURS_FLAG);
+      format.addOptionWithValue(MINUTES_FLAG);
+      format.addOptionWithValue(SECONDS_FLAG);
     }
 
     @VisibleForTesting
     void setMetadataStore(MetadataStore ms) {
       Preconditions.checkNotNull(ms);
-      this.ms = ms;
+      this.setStore(ms);
     }
 
     @Override
@@ -786,7 +795,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
     }
 
     private long getDeltaComponent(TimeUnit unit, String arg) {
-      String raw = commandFormat.getOptValue(arg);
+      String raw = getCommandFormat().getOptValue(arg);
       if (raw == null || raw.isEmpty()) {
         return 0;
       }
@@ -827,7 +836,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
       long now = System.currentTimeMillis();
       long divide = now - delta;
 
-      ms.prune(divide);
+      getStore().prune(divide);
 
       out.flush();
       return SUCCESS;
@@ -839,15 +848,15 @@ public abstract class S3GuardTool extends Configured implements Tool {
     }
   }
 
-  private static S3GuardTool cmd;
+  private static S3GuardTool command;
 
   private static void printHelp() {
-    if (cmd == null) {
+    if (command == null) {
       System.err.println("Usage: hadoop " + USAGE);
       System.err.println("\tperform S3Guard metadata store " +
           "administrative commands.");
     } else {
-      System.err.println("Usage: hadoop " + cmd.getUsage());
+      System.err.println("Usage: hadoop " + command.getUsage());
     }
     System.err.println();
     System.err.println(COMMON_USAGE);
@@ -874,25 +883,25 @@ public abstract class S3GuardTool extends Configured implements Tool {
     final String subCommand = otherArgs[0];
     switch (subCommand) {
     case Init.NAME:
-      cmd = new Init(conf);
+      command = new Init(conf);
       break;
     case Destroy.NAME:
-      cmd = new Destroy(conf);
+      command = new Destroy(conf);
       break;
     case Import.NAME:
-      cmd = new Import(conf);
+      command = new Import(conf);
       break;
     case Diff.NAME:
-      cmd = new Diff(conf);
+      command = new Diff(conf);
       break;
     case Prune.NAME:
-      cmd = new Prune(conf);
+      command = new Prune(conf);
       break;
     default:
       printHelp();
       return INVALID_ARGUMENT;
     }
-    return ToolRunner.run(conf, cmd, otherArgs);
+    return ToolRunner.run(conf, command, otherArgs);
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.Test;
 
 import java.io.FileNotFoundException;
+import java.util.concurrent.Callable;
 
 /**
  * Tests behavior of a FileNotFound error that happens after open(), i.e. on
@@ -45,11 +46,17 @@ public class ITestS3ADelayedFNF extends AbstractS3ATestBase {
     Path p = path("some-file");
     ContractTestUtils.createFile(fs, p, false, new byte[] {20, 21, 22});
 
-    FSDataInputStream in = fs.open(p);
+    final FSDataInputStream in = fs.open(p);
     assertDeleted(p, false);
 
     // This should fail since we deleted after the open.
-    LambdaTestUtils.intercept(FileNotFoundException.class, () -> in.read());
+    LambdaTestUtils.intercept(FileNotFoundException.class,
+        new Callable<Integer>() {
+          @Override
+          public Integer call() throws Exception {
+            return in.read();
+          }
+        });
   }
 
 }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java

@@ -27,11 +27,10 @@ import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystemContractBaseTest;
 import org.apache.hadoop.fs.Path;
+
 import static org.junit.Assume.*;
 import static org.junit.Assert.*;
 
-import java.io.IOException;
-
 /**
  *  Tests a live S3 system. If your keys and bucket aren't specified, all tests
  *  are marked as passed.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java

@@ -26,11 +26,13 @@ import org.apache.hadoop.test.LambdaTestUtils;
 
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
 import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.concurrent.Callable;
 
 /**
  * Tests of the S3A FileSystem which don't have a specific home and can share
@@ -62,7 +64,7 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
 
   @Test
   public void testPutObjectDirect() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
+    final S3AFileSystem fs = getFileSystem();
     ObjectMetadata metadata = fs.newObjectMetadata(-1);
     metadata.setContentLength(-1);
     Path path = path("putDirect");
@@ -71,7 +73,12 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
         new ByteArrayInputStream("PUT".getBytes()),
         metadata);
     LambdaTestUtils.intercept(IllegalStateException.class,
-        () -> fs.putObjectDirect(put));
+        new Callable<PutObjectResult>() {
+          @Override
+          public PutObjectResult call() throws Exception {
+            return fs.putObjectDirect(put);
+          }
+        });
     assertPathDoesNotExist("put object was created", path);
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java

@@ -124,13 +124,14 @@ public class DynamoDBLocalClientFactory extends Configured
     try {
       // Start an in-memory local DynamoDB instance
       final String port = String.valueOf(ServerSocketUtil.getPort(0, 100));
+      ddbEndpoint = "http://localhost:" + port;
       dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs(
           new String[]{"-inMemory", "-port", port});
       dynamoDBLocalServer.start();
-      ddbEndpoint = "http://localhost:" + port;
       LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint);
-    } catch (Throwable t) {
-      String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint;
+    } catch (Exception t) {
+      String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint
+          + " " + t;
       LOG.error(msg, t);
       throw new IOException(msg, t);
     }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java

@@ -80,7 +80,7 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
 
   @Test
   public void testConcurrentTableCreations() throws Exception {
-    Configuration conf = getConfiguration();
+    final Configuration conf = getConfiguration();
     Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
         conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
             Constants.S3GUARD_METASTORE_DYNAMO));
@@ -121,7 +121,8 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
           @Override
           public Exception call() throws Exception {
 
-            ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+            ContractTestUtils.NanoTimer timer =
+                new ContractTestUtils.NanoTimer();
 
             Exception result = null;
             try (DynamoDBMetadataStore store = new DynamoDBMetadataStore()) {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -59,9 +59,9 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
   @Test
   public void testInvalidRegion() throws Exception {
     final String testTableName = "testInvalidRegion" + new Random().nextInt();
-    String testRegion = "invalidRegion";
+    final String testRegion = "invalidRegion";
     // Initialize MetadataStore
-    Init initCmd = new Init(getFileSystem().getConf());
+    final Init initCmd = new Init(getFileSystem().getConf());
     LambdaTestUtils.intercept(IOException.class,
         new Callable<String>() {
           @Override

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
 
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
 
-
 /**
  * Test S3Guard related CLI commands against a LocalMetadataStore.
  */
@@ -65,7 +64,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
     }
 
     S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
-    cmd.setMetadataStore(ms);
+    cmd.setStore(ms);
 
     expectSuccess("Import command did not exit successfully - see output",
         cmd,
@@ -111,7 +110,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
     ByteArrayOutputStream buf = new ByteArrayOutputStream();
     PrintStream out = new PrintStream(buf);
     Diff cmd = new Diff(fs.getConf());
-    cmd.setMetadataStore(ms);
+    cmd.setStore(ms);
     assertEquals("Diff command did not exit successfully - see output", SUCCESS,
         cmd.run(new String[]{"diff", "-meta", "local://metadata",
             testPath.toString()}, out));

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 import org.junit.Rule;
@@ -63,7 +63,8 @@ public class TestDirListingMetadata {
   public void testEmptyListing() {
     Path path = new Path("/path");
     DirListingMetadata meta = new DirListingMetadata(path,
-        Collections.emptyList(), false);
+        new ArrayList<PathMetadata>(0),
+        false);
     assertEquals(path, meta.getPath());
     assertNotNull(meta.getListing());
     assertTrue(meta.getListing().isEmpty());
@@ -294,7 +295,8 @@ public class TestDirListingMetadata {
     PathMetadata pathMeta2 = new PathMetadata(
         new S3AFileStatus(true, new Path(parent, "dir2"), TEST_OWNER));
     PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192, TEST_OWNER));
+        new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192,
+            TEST_OWNER));
     List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
     return new DirListingMetadata(parent, listing, false);
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.services.dynamodbv2.document.DynamoDB;
@@ -105,7 +106,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
           + "server is not configured correctly. ";
       LOG.error(msg, e);
       // fail fast if the DynamoDBLocal server can not work
-      fail(msg + e.getMessage());
+      throw e;
     }
   }
 
@@ -274,14 +275,14 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     ms.put(new PathMetadata(basicFileStatus(oldDir, 0, true)));
     ms.put(new PathMetadata(basicFileStatus(newDir, 0, true)));
 
-    final Collection<PathMetadata> oldMetas =
-        numDelete < 0 ? null : new ArrayList<>(numDelete);
+    final List<PathMetadata> oldMetas =
+        numDelete < 0 ? null : new ArrayList<PathMetadata>(numDelete);
     for (int i = 0; i < numDelete; i++) {
       oldMetas.add(new PathMetadata(
           basicFileStatus(new Path(oldDir, "child" + i), i, true)));
     }
-    final Collection<PathMetadata> newMetas =
-        numPut < 0 ? null : new ArrayList<>(numPut);
+    final List<PathMetadata> newMetas =
+        numPut < 0 ? null : new ArrayList<PathMetadata>(numPut);
     for (int i = 0; i < numPut; i++) {
       newMetas.add(new PathMetadata(
           basicFileStatus(new Path(newDir, "child" + i), i, false)));
@@ -336,9 +337,14 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
   @Test
   public void testItemLacksVersion() throws Throwable {
     intercept(IOException.class, E_NOT_VERSION_MARKER,
-        () -> verifyVersionCompatibility("table",
-            new Item().withPrimaryKey(
-                createVersionMarkerPrimaryKey(VERSION_MARKER))));
+        new VoidCallable() {
+          @Override
+          public void call() throws Exception {
+            verifyVersionCompatibility("table",
+                new Item().withPrimaryKey(
+                    createVersionMarkerPrimaryKey(VERSION_MARKER)));
+          }
+        });
   }
 
   /**
@@ -358,7 +364,12 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
 
     // create existing table
     intercept(IOException.class, E_NO_VERSION_MARKER,
-        () -> ddbms.initTable());
+        new VoidCallable() {
+          @Override
+          public void call() throws Exception {
+            ddbms.initTable();
+          }
+        });
 
     conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
   }
@@ -378,7 +389,13 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
     table.putItem(v200);
 
     // create existing table
-    intercept(IOException.class, E_INCOMPATIBLE_VERSION, ddbms::initTable);
+    intercept(IOException.class, E_INCOMPATIBLE_VERSION,
+        new VoidCallable() {
+          @Override
+          public void call() throws Exception {
+            ddbms.initTable();
+          }
+        });
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
+import java.util.concurrent.Callable;
 
 import com.amazonaws.services.dynamodbv2.document.Item;
 import com.amazonaws.services.dynamodbv2.document.KeyAttribute;
@@ -193,7 +194,12 @@ public class TestPathMetadataDynamoDBTranslation extends Assert {
   @Test
   public void testPathToKey() throws Exception {
     LambdaTestUtils.intercept(IllegalArgumentException.class,
-        () -> pathToKey(new Path("/")));
+        new Callable<PrimaryKey>() {
+          @Override
+          public PrimaryKey call() throws Exception {
+            return pathToKey(new Path("/"));
+          }
+        });
     doTestPathToKey(TEST_DIR_PATH);
     doTestPathToKey(TEST_FILE_PATH);
   }