HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)

Followup to the original HADOOP-18582.

Temporary path cleanup is re-enabled for -append jobs
as these will create temporary files when creating or overwriting files.

Contributed by Ayush Saxena

hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java

@@ -156,10 +156,7 @@ public class CopyCommitter extends FileOutputCommitter {
 
     final boolean directWrite = conf.getBoolean(
         DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
-    final boolean append = conf.getBoolean(
-        DistCpOptionSwitch.APPEND.getConfigLabel(), false);
-    final boolean useTempTarget = !append && !directWrite;
-    if (!useTempTarget) {
+    if (directWrite) {
       return;
     }
 

hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java

@@ -586,13 +586,11 @@ public class TestCopyCommitter {
 
   @Test
   public void testCommitWithCleanupTempFiles() throws IOException {
-    testCommitWithCleanup(true, false);
-    testCommitWithCleanup(false, true);
-    testCommitWithCleanup(true, true);
-    testCommitWithCleanup(false, false);
+    testCommitWithCleanup(true);
+    testCommitWithCleanup(false);
   }
 
-  private void testCommitWithCleanup(boolean append, boolean directWrite)throws IOException {
+  private void testCommitWithCleanup(boolean directWrite) throws IOException {
     TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
     JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
     JobContext jobContext = new JobContextImpl(
@@ -611,7 +609,7 @@ public class TestCopyCommitter {
       DistCpOptions options = new DistCpOptions.Builder(
           Collections.singletonList(new Path(sourceBase)),
           new Path("/out"))
-          .withAppend(append)
+          .withAppend(true)
           .withSyncFolder(true)
           .withDirectWrite(directWrite)
           .build();
@@ -631,7 +629,7 @@ public class TestCopyCommitter {
           null, taskAttemptContext);
       committer.commitJob(jobContext);
 
-      if (append || directWrite) {
+      if (directWrite) {
         ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
             tempFilePath);
       } else {