HADOOP-3011. Prohibit distcp from overwriting directories on the

destination filesystem with files.



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.16@638093 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -1,5 +1,12 @@
 Hadoop Change Log
 
+Release 0.16.2 - Unreleased
+
+  BUG FIXES
+
+    HADOOP-3011. Prohibit distcp from overwriting directories on the
+    destination filesystem with files. (cdouglas)
+
 Release 0.16.1 - 2008-03-13
 
   INCOMPATIBLE CHANGES

src/java/org/apache/hadoop/util/CopyFiles.java

@@ -327,8 +327,16 @@ public class CopyFiles implements Tool {
       else {
         if (totfiles == 1) {
           // Copying a single file; use dst path provided by user as destination
-          // rather than destination directory
-          absdst = absdst.getParent();
+          // rather than destination directory, if a file
+          Path dstparent = absdst.getParent();
+          if (!(destFileSys.exists(dstparent) &&
+                destFileSys.getFileStatus(dstparent).isDir())) {
+            absdst = dstparent;
+          }
+        }
+        if (destFileSys.exists(absdst) &&
+            destFileSys.getFileStatus(absdst).isDir()) {
+          throw new IOException(absdst + " is a directory");
         }
         rename(destFileSys, tmpfile, absdst);
       }

src/test/org/apache/hadoop/fs/TestCopyFiles.java

@@ -57,7 +57,9 @@ public class TestCopyFiles extends TestCase {
     private long seed = 0L;
 
     MyFile() {
-      int nLevels = gen.nextInt(MAX_LEVELS);
+      this(gen.nextInt(MAX_LEVELS));
+    }
+    MyFile(int nLevels) {
       String xname = "";
       if (nLevels != 0) {
         int[] levels = new int[nLevels];
@@ -101,8 +103,9 @@ public class TestCopyFiles extends TestCase {
     return files;
   }
 
-  static MyFile createFile(Path root, FileSystem fs) throws IOException {
-    MyFile f = new MyFile();
+  static MyFile createFile(Path root, FileSystem fs, int levels)
+      throws IOException {
+    MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
     Path p = new Path(root, f.getName());
     FSDataOutputStream out = fs.create(p);
     byte[] toWrite = new byte[f.getSize()];
@@ -113,6 +116,10 @@ public class TestCopyFiles extends TestCase {
     return f;
   }
 
+  static MyFile createFile(Path root, FileSystem fs) throws IOException {
+    return createFile(root, fs, -1);
+  }
+
   /** check if the files have been copied correctly. */
   private static boolean checkFiles(String fsname, String topdir, MyFile[] files) 
     throws IOException {
@@ -413,6 +420,25 @@ public class TestCopyFiles extends TestCase {
                         "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
       assertTrue("Source and destination directories do not match.",
           checkFiles("local", TEST_ROOT_DIR+"/dest2", files));     
+      //copy single file to existing dir
+      deldir("local", TEST_ROOT_DIR+"/dest2");
+      fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
+      MyFile[] files2 = {createFile(root, fs, 0)};
+      String sname = files2[0].getName();
+      ToolRunner.run(new CopyFiles(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));     
+      updateFiles("local", TEST_ROOT_DIR+"/srcdat", files2, 1);
+      //copy single file to existing dir w/ dst name conflict
+      ToolRunner.run(new CopyFiles(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));     
     }
     finally {
       deldir("local", TEST_ROOT_DIR+"/destdat");