|
@@ -26,9 +26,7 @@ import java.io.IOException;
|
|
|
import java.io.InputStreamReader;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Arrays;
|
|
|
-import java.util.HashSet;
|
|
|
import java.util.List;
|
|
|
-import java.util.Set;
|
|
|
import java.util.concurrent.Callable;
|
|
|
|
|
|
import org.apache.hadoop.test.LambdaTestUtils;
|
|
@@ -38,7 +36,6 @@ import org.junit.Test;
|
|
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
import org.apache.hadoop.fs.s3a.S3AFileSystem;
|
|
|
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
|
|
|
|
|
|
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*;
|
|
|
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
|
|
@@ -90,71 +87,6 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
|
|
|
// assertTrue(children.isAuthoritative());
|
|
|
}
|
|
|
|
|
|
- @Test
|
|
|
- public void testDiffCommand() throws Exception {
|
|
|
- S3AFileSystem fs = getFileSystem();
|
|
|
- MetadataStore ms = getMetadataStore();
|
|
|
- Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
|
|
|
- Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
|
|
|
-
|
|
|
- Path testPath = path("test-diff");
|
|
|
- mkdirs(testPath, true, true);
|
|
|
-
|
|
|
- Path msOnlyPath = new Path(testPath, "ms_only");
|
|
|
- mkdirs(msOnlyPath, false, true);
|
|
|
- filesOnMS.add(msOnlyPath);
|
|
|
- for (int i = 0; i < 5; i++) {
|
|
|
- Path file = new Path(msOnlyPath, String.format("file-%d", i));
|
|
|
- createFile(file, false, true);
|
|
|
- filesOnMS.add(file);
|
|
|
- }
|
|
|
-
|
|
|
- Path s3OnlyPath = new Path(testPath, "s3_only");
|
|
|
- mkdirs(s3OnlyPath, true, false);
|
|
|
- filesOnS3.add(s3OnlyPath);
|
|
|
- for (int i = 0; i < 5; i++) {
|
|
|
- Path file = new Path(s3OnlyPath, String.format("file-%d", i));
|
|
|
- createFile(file, true, false);
|
|
|
- filesOnS3.add(file);
|
|
|
- }
|
|
|
-
|
|
|
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
|
|
- Diff cmd = new Diff(fs.getConf());
|
|
|
- cmd.setStore(ms);
|
|
|
- exec(cmd, buf, "diff", "-meta", LOCAL_METADATA,
|
|
|
- testPath.toString());
|
|
|
-
|
|
|
- Set<Path> actualOnS3 = new HashSet<>();
|
|
|
- Set<Path> actualOnMS = new HashSet<>();
|
|
|
- boolean duplicates = false;
|
|
|
- try (BufferedReader reader =
|
|
|
- new BufferedReader(new InputStreamReader(
|
|
|
- new ByteArrayInputStream(buf.toByteArray())))) {
|
|
|
- String line;
|
|
|
- while ((line = reader.readLine()) != null) {
|
|
|
- String[] fields = line.split("\\s");
|
|
|
- assertEquals("[" + line + "] does not have enough fields",
|
|
|
- 4, fields.length);
|
|
|
- String where = fields[0];
|
|
|
- Path path = new Path(fields[3]);
|
|
|
- if (Diff.S3_PREFIX.equals(where)) {
|
|
|
- duplicates = duplicates || actualOnS3.contains(path);
|
|
|
- actualOnS3.add(path);
|
|
|
- } else if (Diff.MS_PREFIX.equals(where)) {
|
|
|
- duplicates = duplicates || actualOnMS.contains(path);
|
|
|
- actualOnMS.add(path);
|
|
|
- } else {
|
|
|
- fail("Unknown prefix: " + where);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- String actualOut = buf.toString();
|
|
|
- assertEquals("Mismatched metadata store outputs: " + actualOut,
|
|
|
- filesOnMS, actualOnMS);
|
|
|
- assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
|
|
|
- assertFalse("Diff contained duplicates", duplicates);
|
|
|
- }
|
|
|
-
|
|
|
@Test
|
|
|
public void testDestroyBucketExistsButNoTable() throws Throwable {
|
|
|
run(Destroy.NAME,
|