HADOOP-17328. LazyPersist Overwrite fails in direct write mode. (#2413)

(cherry picked from commit 872440610f066d4b12c9f93c05477848a260b21f)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java

@@ -54,6 +54,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
 import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
 
 /**
  * Provides: argument processing to ensure the destination is valid
@@ -515,7 +516,8 @@ abstract class CommandWithDestination extends FsCommand {
           defaultBlockSize = getDefaultBlockSize(item.path);
         }
 
-        EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE, LAZY_PERSIST);
+        EnumSet<CreateFlag> createFlags =
+            EnumSet.of(CREATE, LAZY_PERSIST, OVERWRITE);
         return create(item.path,
                       FsPermission.getFileDefault().applyUMask(
                           FsPermission.getUMask(getConf())),

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java

@@ -23,6 +23,7 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -697,4 +698,27 @@ public class TestFsShellCopy {
       lfs.setPermission(src, new FsPermission((short)0755));
     }
   }
+
+  @Test
+  public void testLazyPersistDirectOverwrite() throws Exception {
+    Path testRoot = new Path(testRootDir, "testLazyPersistDirectOverwrite");
+    try {
+      lfs.delete(testRoot, true);
+      lfs.mkdirs(testRoot);
+      Path filePath = new Path(testRoot, new Path("srcFile"));
+      lfs.create(filePath).close();
+      // Put with overwrite in direct mode.
+      String[] argv =
+          new String[] {"-put", "-f", "-l", "-d", filePath.toString(),
+              filePath.toString()};
+      assertEquals(0, shell.run(argv));
+
+      // Put without overwrite in direct mode shouldn't be success.
+      argv = new String[] {"-put", "-l", "-d", filePath.toString(),
+          filePath.toString()};
+      assertNotEquals(0, shell.run(argv));
+    } finally {
+      lfs.delete(testRoot, true);
+    }
+  }
 }