|
@@ -48,7 +48,11 @@ import org.apache.hadoop.tools.GlobbedCopyListing;
|
|
import org.apache.hadoop.tools.util.DistCpUtils;
|
|
import org.apache.hadoop.tools.util.DistCpUtils;
|
|
import org.apache.hadoop.tools.util.TestDistCpUtils;
|
|
import org.apache.hadoop.tools.util.TestDistCpUtils;
|
|
import org.apache.hadoop.security.Credentials;
|
|
import org.apache.hadoop.security.Credentials;
|
|
-import org.junit.*;
|
|
|
|
|
|
+import org.junit.jupiter.api.AfterAll;
|
|
|
|
+import org.junit.jupiter.api.AfterEach;
|
|
|
|
+import org.junit.jupiter.api.BeforeAll;
|
|
|
|
+import org.junit.jupiter.api.BeforeEach;
|
|
|
|
+import org.junit.jupiter.api.Test;
|
|
|
|
|
|
import java.io.IOException;
|
|
import java.io.IOException;
|
|
import java.util.*;
|
|
import java.util.*;
|
|
@@ -82,7 +86,7 @@ public class TestCopyCommitter {
|
|
return job;
|
|
return job;
|
|
}
|
|
}
|
|
|
|
|
|
- @BeforeClass
|
|
|
|
|
|
+ @BeforeAll
|
|
public static void create() throws IOException {
|
|
public static void create() throws IOException {
|
|
clusterConfig = getJobForClient().getConfiguration();
|
|
clusterConfig = getJobForClient().getConfiguration();
|
|
clusterConfig.setLong(
|
|
clusterConfig.setLong(
|
|
@@ -97,14 +101,14 @@ public class TestCopyCommitter {
|
|
.build();
|
|
.build();
|
|
}
|
|
}
|
|
|
|
|
|
- @AfterClass
|
|
|
|
|
|
+ @AfterAll
|
|
public static void destroy() {
|
|
public static void destroy() {
|
|
if (cluster != null) {
|
|
if (cluster != null) {
|
|
cluster.shutdown();
|
|
cluster.shutdown();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- @Before
|
|
|
|
|
|
+ @BeforeEach
|
|
public void createMetaFolder() throws IOException {
|
|
public void createMetaFolder() throws IOException {
|
|
config = new Configuration(clusterConfig);
|
|
config = new Configuration(clusterConfig);
|
|
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
|
|
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
|
|
@@ -112,12 +116,12 @@ public class TestCopyCommitter {
|
|
cluster.getFileSystem().mkdirs(meta);
|
|
cluster.getFileSystem().mkdirs(meta);
|
|
}
|
|
}
|
|
|
|
|
|
- @After
|
|
|
|
|
|
+ @AfterEach
|
|
public void cleanupMetaFolder() throws IOException {
|
|
public void cleanupMetaFolder() throws IOException {
|
|
Path meta = new Path("/meta");
|
|
Path meta = new Path("/meta");
|
|
if (cluster.getFileSystem().exists(meta)) {
|
|
if (cluster.getFileSystem().exists(meta)) {
|
|
cluster.getFileSystem().delete(meta, true);
|
|
cluster.getFileSystem().delete(meta, true);
|
|
- Assert.fail("Expected meta folder to be deleted");
|
|
|
|
|
|
+ fail("Expected meta folder to be deleted");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -129,11 +133,11 @@ public class TestCopyCommitter {
|
|
taskAttemptContext.getTaskAttemptID().getJobID());
|
|
taskAttemptContext.getTaskAttemptID().getJobID());
|
|
OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
|
|
OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
- Assert.assertEquals("Commit Successful", taskAttemptContext.getStatus());
|
|
|
|
|
|
+ assertEquals("Commit Successful", taskAttemptContext.getStatus());
|
|
|
|
|
|
//Test for idempotent commit
|
|
//Test for idempotent commit
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
- Assert.assertEquals("Commit Successful", taskAttemptContext.getStatus());
|
|
|
|
|
|
+ assertEquals("Commit Successful", taskAttemptContext.getStatus());
|
|
}
|
|
}
|
|
|
|
|
|
@Test
|
|
@Test
|
|
@@ -412,12 +416,12 @@ public class TestCopyCommitter {
|
|
|
|
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
verifyFoldersAreInSync(fs, targetBase, sourceBase);
|
|
verifyFoldersAreInSync(fs, targetBase, sourceBase);
|
|
- Assert.assertEquals(4, fs.listStatus(new Path(targetBase)).length);
|
|
|
|
|
|
+ assertEquals(4, fs.listStatus(new Path(targetBase)).length);
|
|
|
|
|
|
//Test for idempotent commit
|
|
//Test for idempotent commit
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
verifyFoldersAreInSync(fs, targetBase, sourceBase);
|
|
verifyFoldersAreInSync(fs, targetBase, sourceBase);
|
|
- Assert.assertEquals(4, fs.listStatus(new Path(targetBase)).length);
|
|
|
|
|
|
+ assertEquals(4, fs.listStatus(new Path(targetBase)).length);
|
|
} finally {
|
|
} finally {
|
|
TestDistCpUtils.delete(fs, "/tmp1");
|
|
TestDistCpUtils.delete(fs, "/tmp1");
|
|
conf.set(DistCpConstants.CONF_LABEL_DELETE_MISSING, "false");
|
|
conf.set(DistCpConstants.CONF_LABEL_DELETE_MISSING, "false");
|
|
@@ -486,7 +490,7 @@ public class TestCopyCommitter {
|
|
assertPathExists(fs, "Final path", new Path(finalPath));
|
|
assertPathExists(fs, "Final path", new Path(finalPath));
|
|
try {
|
|
try {
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
- Assert.fail("Should not be able to atomic-commit to pre-existing path.");
|
|
|
|
|
|
+ fail("Should not be able to atomic-commit to pre-existing path.");
|
|
} catch(Exception exception) {
|
|
} catch(Exception exception) {
|
|
assertPathExists(fs, "Work path", new Path(workPath));
|
|
assertPathExists(fs, "Work path", new Path(workPath));
|
|
assertPathExists(fs, "Final path", new Path(finalPath));
|
|
assertPathExists(fs, "Final path", new Path(finalPath));
|
|
@@ -558,17 +562,17 @@ public class TestCopyCommitter {
|
|
try {
|
|
try {
|
|
committer.commitJob(jobContext);
|
|
committer.commitJob(jobContext);
|
|
if (!skipCrc) {
|
|
if (!skipCrc) {
|
|
- Assert.fail("Expected commit to fail");
|
|
|
|
|
|
+ fail("Expected commit to fail");
|
|
}
|
|
}
|
|
Path sourcePath = new Path(sourceBase + srcFilename);
|
|
Path sourcePath = new Path(sourceBase + srcFilename);
|
|
CopyListingFileStatus sourceCurrStatus =
|
|
CopyListingFileStatus sourceCurrStatus =
|
|
- new CopyListingFileStatus(fs.getFileStatus(sourcePath));
|
|
|
|
- Assert.assertEquals("Checksum should not be equal",
|
|
|
|
- CopyMapper.ChecksumComparison.FALSE,
|
|
|
|
|
|
+ new CopyListingFileStatus(fs.getFileStatus(sourcePath));
|
|
|
|
+ assertEquals(CopyMapper.ChecksumComparison.FALSE,
|
|
DistCpUtils.checksumsAreEqual(
|
|
DistCpUtils.checksumsAreEqual(
|
|
fs, new Path(sourceBase + srcFilename), null,
|
|
fs, new Path(sourceBase + srcFilename), null,
|
|
fs, new Path(targetBase + srcFilename),
|
|
fs, new Path(targetBase + srcFilename),
|
|
- sourceCurrStatus.getLen()));
|
|
|
|
|
|
+ sourceCurrStatus.getLen()),
|
|
|
|
+ "Checksum should not be equal");
|
|
} catch(IOException exception) {
|
|
} catch(IOException exception) {
|
|
if (skipCrc) {
|
|
if (skipCrc) {
|
|
LOG.error("Unexpected exception is found", exception);
|
|
LOG.error("Unexpected exception is found", exception);
|
|
@@ -729,7 +733,7 @@ public class TestCopyCommitter {
|
|
for (FileStatus status : fStatus) {
|
|
for (FileStatus status : fStatus) {
|
|
if (status.isDirectory()) {
|
|
if (status.isDirectory()) {
|
|
stack.push(status.getPath());
|
|
stack.push(status.getPath());
|
|
- Assert.assertEquals(sourcePerm, status.getPermission());
|
|
|
|
|
|
+ assertEquals(sourcePerm, status.getPermission());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -746,10 +750,10 @@ public class TestCopyCommitter {
|
|
while (sourceReader.next(srcRelPath, srcFileStatus)) {
|
|
while (sourceReader.next(srcRelPath, srcFileStatus)) {
|
|
Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
|
|
Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
|
|
FileStatus targetStatus = fs.getFileStatus(targetFile);
|
|
FileStatus targetStatus = fs.getFileStatus(targetFile);
|
|
- Assert.assertEquals(srcFileStatus.getModificationTime(),
|
|
|
|
- targetStatus.getModificationTime());
|
|
|
|
- Assert.assertEquals(srcFileStatus.getAccessTime(),
|
|
|
|
- targetStatus.getAccessTime());
|
|
|
|
|
|
+ assertEquals(srcFileStatus.getModificationTime(),
|
|
|
|
+ targetStatus.getModificationTime());
|
|
|
|
+ assertEquals(srcFileStatus.getAccessTime(),
|
|
|
|
+ targetStatus.getAccessTime());
|
|
}
|
|
}
|
|
} finally {
|
|
} finally {
|
|
IOUtils.closeStream(sourceReader);
|
|
IOUtils.closeStream(sourceReader);
|