HADOOP-13995. s3guard cli: make tests easier to run and address failure. Contributed by Sean Mackrory.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -100,36 +100,43 @@ public abstract class S3GuardTool extends Configured implements Tool {
    * @throws IOException on I/O errors.
    */
   boolean parseDynamoDBEndPoint(List<String> paths) throws IOException {
-    // Validate parameters.
-    boolean hasMsURI = commandFormat.getOptValue("m") != null &&
-        !commandFormat.getOptValue("m").isEmpty();
+    Configuration conf = getConf();
+    String fromCli = commandFormat.getOptValue("e");
+    String fromConf = conf.get(S3GUARD_DDB_ENDPOINT_KEY);
     boolean hasS3Path = !paths.isEmpty();
-    if (hasMsURI && hasS3Path) {
-      System.out.println("Must specify either -m URI or s3a://bucket");
-      return false;
-    } else if (hasMsURI) {
-      Configuration conf = getConf();
-      String endpoint = conf.get(S3GUARD_DDB_ENDPOINT_KEY);
-      String param = commandFormat.getOptValue("e");  // endpoint param
-      if ((endpoint == null || endpoint.isEmpty())
-          && (param == null || param.isEmpty())) {
-        System.out.printf("Must specify -e ENDPOINT or %s in conf.%n",
-            S3GUARD_DDB_ENDPOINT_KEY);
+
+    if (fromCli != null) {
+      if (fromCli.isEmpty()) {
+        System.out.println("No endpoint provided with -e flag");
+        return false;
+      }
+      if (hasS3Path) {
+        System.out.println("Providing both an S3 path and the -e flag is not " +
+            "supported. If you need to specify an endpoint for a different " +
+            "region than the S3 bucket, configure " + S3GUARD_DDB_ENDPOINT_KEY);
         return false;
       }
+      conf.set(S3GUARD_DDB_ENDPOINT_KEY, fromCli);
+      return true;
+    }
 
-      if (param != null && !param.isEmpty()) {
-        conf.set(S3GUARD_DDB_ENDPOINT_KEY, param);
+    if (fromConf != null) {
+      if (fromConf.isEmpty()) {
+        System.out.printf("No endpoint provided with config %s, %n",
+            S3GUARD_DDB_ENDPOINT_KEY);
+        return false;
       }
-    } else if (hasS3Path) {
-      // This CLI has a valid S3 path, so it uses S3AFileSystem instance
-      // to configure metadata store later.
+      return true;
+    }
+
+    if (hasS3Path) {
       String s3Path = paths.get(0);
       initS3AFileSystem(s3Path);
-    } else {
-      return false;
+      return true;
     }
-    return true;
+
+    System.out.println("No endpoint found from -e flag, config, or S3 bucket");
+    return false;
   }
 
   /**
@@ -142,6 +149,12 @@ public abstract class S3GuardTool extends Configured implements Tool {
     if (ms != null) {
       return ms;
     }
+    Configuration conf;
+    if (s3a == null) {
+      conf = getConf();
+    } else {
+      conf = s3a.getConf();
+    }
     String metaURI = commandFormat.getOptValue("m");
     if (metaURI != null && !metaURI.isEmpty()) {
       URI uri = URI.create(metaURI);
@@ -153,8 +166,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
         break;
       case "dynamodb":
         ms = new DynamoDBMetadataStore();
-        getConf().set(S3GUARD_DDB_TABLE_NAME_KEY, uri.getAuthority());
-        getConf().setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, create);
+        conf.set(S3GUARD_DDB_TABLE_NAME_KEY, uri.getAuthority());
+        conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, create);
         break;
       default:
         throw new IOException(
@@ -164,11 +177,11 @@ public abstract class S3GuardTool extends Configured implements Tool {
       // CLI does not specify metadata store URI, it uses default metadata store
       // DynamoDB instead.
       ms = new DynamoDBMetadataStore();
-      getConf().setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, create);
+      conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, create);
     }
 
     if (s3a == null) {
-      ms.initialize(getConf());
+      ms.initialize(conf);
     } else {
       ms.initialize(s3a);
     }
@@ -220,7 +233,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
   static class InitMetadata extends S3GuardTool {
     private static final String NAME = "init";
     private static final String USAGE = NAME +
-        " [-r UNIT] [-w UNIT] [-e ENDPOINT -m URI|s3a://bucket]";
+        " [-r UNIT] [-w UNIT] -m URI ( -e ENDPOINT | s3a://bucket )";
 
     InitMetadata(Configuration conf) {
       super(conf);
@@ -252,6 +265,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
 
       // Validate parameters.
       if (!parseDynamoDBEndPoint(paths)) {
+        System.out.println(USAGE);
         return INVALID_ARGUMENT;
       }
       initMetadataStore(true);
@@ -265,7 +279,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
   static class DestroyMetadata extends S3GuardTool {
     private static final String NAME = "destroy";
     private static final String USAGE =
-        NAME + " [-e ENDPOINT -m URI|s3a://bucket]";
+        NAME + " -m URI ( -e ENDPOINT | s3a://bucket )";
 
     DestroyMetadata(Configuration conf) {
       super(conf);
@@ -390,8 +404,16 @@ public abstract class S3GuardTool extends Configured implements Tool {
         throw new IOException(e);
       }
       String filePath = uri.getPath();
+      if (filePath.isEmpty()) {
+        // If they specify a naked S3 URI (e.g. s3a://bucket), we'll consider
+        // root to be the path
+        filePath = "/";
+      }
       Path path = new Path(filePath);
       S3AFileStatus status = s3a.getFileStatus(path);
+
+      initMetadataStore(false);
+
       if (status.isFile()) {
         PathMetadata meta = new PathMetadata(status);
         ms.put(meta);

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -83,8 +83,8 @@ Note that the Null Metadata store can be explicitly requested if desired.
 
 ```xml
 <property>
-  <name>fs.s3a.metadatastore.impl</name>
-  <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
+    <name>fs.s3a.metadatastore.impl</name>
+    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
 </property>
 ```
 
@@ -135,6 +135,23 @@ choose our own table name:
 </property>
 ```
 
+You may also wish to specify the endpoint to use for DynamoDB. In AWS, the
+endpoint should be matched to the region the table lives in. If an endpoint
+is not configured, it will be assume that it is in the same region as the S3
+bucket. A list of regions and endpoints for the DynamoDB service can be found in
+[Amazon's documentation](http://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region).
+
+```xml
+<property>
+  <name>fs.s3a.s3guard.ddb.endpoint</name>
+  <value>dynamodb.us-west-1.amazonaws.com</value>
+  <description>
+    Endpoint to use for DynamoDB requests. The endpoint should be matched to the
+    region the metastore database will be in.
+  </description>
+</property>
+```
+
 Next, you can choose whether or not the table will be automatically created
 (if it doesn't already exist).  If we want this feature, we can set the
 following parameter to true.
@@ -193,7 +210,52 @@ with S3Guard.
 
 ## S3Guard Command Line Interface (CLI)
 
-TODO: TBD
+Note that in some cases an endpoint or a s3a:// URI can be provided.
+
+Metadata store URIs include a scheme that designates the backing store. For
+example (e.g. dynamodb://&lt;table_name&gt;). As documented above, endpoints can
+be inferred if the URI to an existing bucket is provided.
+
+### Init
+
+```
+hadoop s3a init -m URI ( -e ENDPOINT | s3a://BUCKET )
+```
+
+Creates and initializes an empty metadata store.
+
+A DynamoDB metadata store can be initialized with additional parameters
+pertaining to (provisioned throughput)[http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ProvisionedThroughput.html]:
+
+```
+[-w PROVISIONED_WRITES] [-r PROVISIONED_READS]
+```
+
+### Import
+
+```
+hadoop s3a import [-m URI] s3a://BUCKET
+```
+
+Pre-populates a metadata store according to the current contents of an S3
+bucket.
+
+### Diff
+
+```
+hadoop s3a diff [-m URI] s3a://BUCKET
+```
+
+Lists discrepancies between a metadata store and bucket. Note that depending on
+how S3Guard is used, certain discrepancies are to be expected.
+
+### Destroy
+
+```
+hadoop s3a destroy [-m URI] ( -e ENDPOINT | s3a://BUCKET )
+```
+
+Deletes a metadata store.
 
 ## Debugging and Error Handling
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java

@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import com.amazonaws.services.dynamodbv2.document.DynamoDB;
+import com.amazonaws.services.dynamodbv2.document.Table;
+import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
+import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
+import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.DestroyMetadata;
+import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.InitMetadata;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Random;
+
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test S3Guard related CLI commands against DynamoDB.
+ */
+public class ITestS3GuardToolDynamoDB extends S3GuardToolTestBase {
+
+  @Override
+  protected MetadataStore newMetadataStore() {
+    return new DynamoDBMetadataStore();
+  }
+
+  // Check the existence of a given DynamoDB table.
+  private static boolean exist(DynamoDB dynamoDB, String tableName) {
+    assertNotNull(dynamoDB);
+    assertNotNull(tableName);
+    assertFalse("empty table name", tableName.isEmpty());
+    try {
+      Table table = dynamoDB.getTable(tableName);
+      table.describe();
+    } catch (ResourceNotFoundException e) {
+      return false;
+    }
+    return true;
+  }
+
+  @Test
+  public void testInitDynamoDBMetadataStore() throws IOException {
+    final String testS3Url = getTestPath("init");
+    String testTableName = "initDynamoDBMetadataStore" +
+        new Random().nextInt();
+
+    InitMetadata cmd = new InitMetadata(getFs().getConf());
+    Table table = null;
+
+    try {
+      String[] args = new String[]{
+          "init", "-m", "dynamodb://" + testTableName, testS3Url
+      };
+      assertEquals("Init command did not exit successfully - see output",
+          SUCCESS, cmd.run(args));
+      // Verify the existence of the dynamodb table.
+      try {
+        MetadataStore ms = getMetadataStore();
+        assertTrue("metadata store should be DynamoDBMetadataStore",
+            ms instanceof DynamoDBMetadataStore);
+        DynamoDBMetadataStore dynamoMs = (DynamoDBMetadataStore) ms;
+        DynamoDB db = dynamoMs.getDynamoDB();
+        table = db.getTable(testTableName);
+        table.describe();
+      } catch (ResourceNotFoundException e) {
+        fail(String.format("DynamoDB table %s does not exist",
+            testTableName));
+      }
+    } finally {
+      // Clean the table.
+      try {
+        if (table != null) {
+          table.delete();
+        }
+      } catch (ResourceNotFoundException e) {
+        // Ignore
+      }
+    }
+  }
+
+  @Test
+  public void testDestroyDynamoDBMetadataStore()
+      throws IOException, InterruptedException {
+    final String testS3Url = getTestPath("destroy");
+    String testTableName = "destroyDynamoDBMetadataStore" +
+        new Random().nextInt();
+
+    S3AFileSystem fs = getFs();
+    DestroyMetadata cmd = new DestroyMetadata(fs.getConf());
+
+    // Pre-alloc DynamoDB table.
+    DynamoDB db = DynamoDBMetadataStore.createDynamoDB(fs);
+    try {
+      Table table = db.getTable(testTableName);
+      table.delete();
+      table.waitForDelete();
+    } catch (ResourceNotFoundException e) {
+      // Ignore.
+    }
+    Collection<KeySchemaElement> elements =
+        PathMetadataDynamoDBTranslation.keySchema();
+    Collection<AttributeDefinition> attrs =
+        PathMetadataDynamoDBTranslation.attributeDefinitions();
+    ProvisionedThroughput pt = new ProvisionedThroughput(100L, 200L);
+    Table table = db.createTable(new CreateTableRequest()
+        .withAttributeDefinitions(attrs)
+        .withKeySchema(elements)
+        .withTableName(testTableName)
+        .withProvisionedThroughput(pt));
+    try {
+      table.waitForActive();
+      assertTrue("Table does not exist", exist(db, testTableName));
+
+      String[] args = new String[]{
+          "destroy", "-m", "dynamodb://" + testTableName, testS3Url
+      };
+      assertEquals("Destroy command did not exit successfully - see output",
+          SUCCESS, cmd.run(args));
+      assertFalse(String.format("%s still exists", testTableName),
+          exist(db, testTableName));
+    } finally {
+      if (table != null) {
+        try {
+          table.delete();
+          table.waitForDelete();
+        } catch (ResourceNotFoundException e) {}
+      }
+    }
+  }
+}

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java

@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Common functionality for S3GuardTool test cases.
+ */
+public abstract class S3GuardToolTestBase {
+
+  protected static final String OWNER = "hdfs";
+
+  private Configuration conf;
+  private MetadataStore ms;
+  private S3AFileSystem fs;
+
+  protected Configuration getConf() {
+    return conf;
+  }
+
+  protected MetadataStore getMetadataStore() {
+    return ms;
+  }
+
+  protected S3AFileSystem getFs() {
+    return fs;
+  }
+
+  /** Get test path of s3. */
+  protected String getTestPath(String path) {
+    return fs.qualify(new Path(path)).toString();
+  }
+
+  protected abstract MetadataStore newMetadataStore();
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    fs = S3ATestUtils.createTestFileSystem(conf);
+
+    ms = newMetadataStore();
+    ms.initialize(fs);
+  }
+
+  @After
+  public void tearDown() {
+  }
+
+  protected void mkdirs(Path path, boolean onS3, boolean onMetadataStore)
+      throws IOException {
+    if (onS3) {
+      fs.mkdirs(path);
+    }
+    if (onMetadataStore) {
+      S3AFileStatus status = new S3AFileStatus(true, path, OWNER);
+      ms.put(new PathMetadata(status));
+    }
+  }
+
+  protected static void putFile(MetadataStore ms, S3AFileStatus f)
+      throws IOException {
+    assertNotNull(f);
+    ms.put(new PathMetadata(f));
+    Path parent = f.getPath().getParent();
+    while (parent != null) {
+      S3AFileStatus dir = new S3AFileStatus(false, parent, f.getOwner());
+      ms.put(new PathMetadata(dir));
+      parent = parent.getParent();
+    }
+  }
+
+  /**
+   * Create file either on S3 or in metadata store.
+   * @param path the file path.
+   * @param onS3 set to true to create the file on S3.
+   * @param onMetadataStore set to true to create the file on the
+   *                        metadata store.
+   * @throws IOException
+   */
+  protected void createFile(Path path, boolean onS3, boolean onMetadataStore)
+      throws IOException {
+    if (onS3) {
+      ContractTestUtils.touch(fs, path);
+    }
+
+    if (onMetadataStore) {
+      S3AFileStatus status = new S3AFileStatus(100L, 10000L,
+          fs.qualify(path), 512L, "hdfs");
+      putFile(ms, status);
+    }
+  }
+}

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java

@@ -18,25 +18,10 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
-import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
-import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.DestroyMetadata;
 import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.InitMetadata;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.BufferedReader;
@@ -45,133 +30,27 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
- * Test S3Guard related CLI commands.
+ * Test S3Guard related CLI commands against a LocalMetadataStore.
  */
-public class TestS3GuardTool {
+public class TestS3GuardTool extends S3GuardToolTestBase {
 
-  private static final String OWNER = "hdfs";
-
-  private Configuration conf;
-  private MetadataStore ms;
-  private S3AFileSystem fs;
-
-  /** Get test path of s3. */
-  private String getTestPath(String path) {
-    return fs.qualify(new Path(path)).toString();
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    conf = new Configuration();
-    fs = S3ATestUtils.createTestFileSystem(conf);
-
-    ms = new LocalMetadataStore();
-    ms.initialize(fs);
-  }
-
-  @After
-  public void tearDown() {
-  }
-
-  // Check the existence of a given DynamoDB table.
-  private static boolean exist(DynamoDB dynamoDB, String tableName) {
-    assertNotNull(dynamoDB);
-    assertNotNull(tableName);
-    assertFalse("empty table name", tableName.isEmpty());
-    try {
-      Table table = dynamoDB.getTable(tableName);
-      table.describe();
-    } catch (ResourceNotFoundException e) {
-      return false;
-    }
-    return true;
-  }
-
-  @Test
-  public void testInitDynamoDBMetadataStore() throws IOException {
-    final String testTableName = "s3guard_test_init_ddb_table";
-    InitMetadata cmd = new InitMetadata(fs.getConf());
-    Table table = null;
-
-    try {
-      String[] args = new String[]{
-          "init", "-m", "dynamodb://" + testTableName,
-      };
-      assertEquals(SUCCESS, cmd.run(args));
-      // Verify the existence of the dynamodb table.
-      try {
-        assertTrue("metadata store should be DynamoDBMetadataStore",
-            ms instanceof DynamoDBMetadataStore);
-        DynamoDBMetadataStore dynamoMs = (DynamoDBMetadataStore) ms;
-        DynamoDB db = dynamoMs.getDynamoDB();
-        table = db.getTable(testTableName);
-        table.describe();
-      } catch (ResourceNotFoundException e) {
-        fail(String.format("DynamoDB table %s does not exist",
-            testTableName));
-      }
-    } finally {
-      // Clean the table.
-      try {
-        if (table != null) {
-          table.delete();
-        }
-      } catch (ResourceNotFoundException e) {
-        // Ignore
-      }
-    }
-  }
-
-  @Test
-  public void testDestroyDynamoDBMetadataStore()
-      throws IOException, InterruptedException {
-    final String testTableName = "s3guard_test_destroy_ddb_table";
-    DestroyMetadata cmd = new DestroyMetadata(fs.getConf());
-
-    // Pre-alloc DynamoDB table.
-    DynamoDB db = DynamoDBMetadataStore.createDynamoDB(fs);
-    try {
-      Table table = db.getTable(testTableName);
-      table.delete();
-      table.waitForDelete();
-    } catch (ResourceNotFoundException e) {
-      // Ignore.
-    }
-    Collection<KeySchemaElement> elements =
-        PathMetadataDynamoDBTranslation.keySchema();
-    Collection<AttributeDefinition> attrs =
-        PathMetadataDynamoDBTranslation.attributeDefinitions();
-    ProvisionedThroughput pt = new ProvisionedThroughput(100L, 200L);
-    Table table = db.createTable(new CreateTableRequest()
-        .withAttributeDefinitions(attrs)
-        .withKeySchema(elements)
-        .withTableName(testTableName)
-        .withProvisionedThroughput(pt));
-    table.waitForActive();
-    assertTrue(exist(db, testTableName));
-
-    String[] args = new String[]{
-        "destroy", "-m", "dynamodb://" + testTableName,
-    };
-    assertEquals(SUCCESS, cmd.run(args));
-    assertFalse(String.format("%s still exists", testTableName),
-        exist(db, testTableName));
+  @Override
+  protected MetadataStore newMetadataStore() {
+    return new LocalMetadataStore();
   }
 
   @Test
   public void testImportCommand() throws IOException {
+    S3AFileSystem fs = getFs();
+    MetadataStore ms = getMetadataStore();
     fs.mkdirs(new Path("/test"));
     Path dir = new Path("/test/a");
     fs.mkdirs(dir);
@@ -185,60 +64,20 @@ public class TestS3GuardTool {
     S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
     cmd.setMetadataStore(ms);
 
-    assertEquals(0, cmd.run(new String[]{"import", getTestPath("/test/a")}));
+    assertEquals("Import command did not exit successfully - see output",
+        SUCCESS, cmd.run(new String[]{"import", getTestPath("/test/a")}));
 
     DirListingMetadata children =
         ms.listChildren(new Path(getTestPath("/test/a")));
-    assertEquals(10, children.getListing().size());
+    assertEquals("Unexpected number of paths imported", 10, children
+        .getListing().size());
     // assertTrue(children.isAuthoritative());
   }
 
-  private void mkdirs(Path path, boolean onS3, boolean onMetadataStore)
-      throws IOException {
-    if (onS3) {
-      fs.mkdirs(path);
-    }
-    if (onMetadataStore) {
-      S3AFileStatus status = new S3AFileStatus(true, path, OWNER);
-      ms.put(new PathMetadata(status));
-    }
-  }
-
-  private static void putFile(MetadataStore ms, S3AFileStatus f)
-      throws IOException {
-    assertNotNull(f);
-    ms.put(new PathMetadata(f));
-    Path parent = f.getPath().getParent();
-    while (parent != null) {
-      S3AFileStatus dir = new S3AFileStatus(false, parent, f.getOwner());
-      ms.put(new PathMetadata(dir));
-      parent = parent.getParent();
-    }
-  }
-
-  /**
-   * Create file either on S3 or in metadata store.
-   * @param path the file path.
-   * @param onS3 set to true to create the file on S3.
-   * @param onMetadataStore set to true to create the file on the
-   *                        metadata store.
-   * @throws IOException
-   */
-  private void createFile(Path path, boolean onS3, boolean onMetadataStore)
-      throws IOException {
-    if (onS3) {
-      ContractTestUtils.touch(fs, path);
-    }
-
-    if (onMetadataStore) {
-      S3AFileStatus status = new S3AFileStatus(100L, 10000L,
-          fs.qualify(path), 512L, "hdfs");
-      putFile(ms, status);
-    }
-  }
-
   @Test
   public void testDiffCommand() throws IOException {
+    S3AFileSystem fs = getFs();
+    MetadataStore ms = getMetadataStore();
     Set<Path> filesOnS3 = new HashSet<>();  // files on S3.
     Set<Path> filesOnMS = new HashSet<>();  // files on metadata store.
 
@@ -267,7 +106,7 @@ public class TestS3GuardTool {
     PrintStream out = new PrintStream(buf);
     Diff cmd = new Diff(fs.getConf());
     cmd.setMetadataStore(ms);
-    assertEquals(SUCCESS,
+    assertEquals("Diff command did not exit successfully - see output", SUCCESS,
         cmd.run(new String[]{"diff", "-m", "local://metadata", testPath}, out));
 
     Set<Path> actualOnS3 = new HashSet<>();