HADOOP-3294. Fix distcp to check the destination length and retry the copy if

it doesn't match the src length. Contributed by Tsz Wo (Nicholas), SZE.



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.17@651456 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -580,6 +580,9 @@ Release 0.17.0 - Unreleased
     HADOOP-3285. Fix input split locality when the splits align to
     fs blocks. (omalley)
 
+    HADOOP-3294. Fix distcp to check the destination length and retry the copy
+    if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via cdouglas)
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

src/java/org/apache/hadoop/util/CopyFiles.java

@@ -382,13 +382,11 @@ public class CopyFiles implements Tool {
         checkAndClose(out);
       }
 
-      final boolean success = cbcopied == srcstat.getLen();
-      if (!success) {
-        final String badlen = "ERROR? copied " + bytesString(cbcopied)
-            + " but expected " + bytesString(srcstat.getLen()) 
-            + " from " + srcstat.getPath();
-        LOG.warn(badlen);
-        outc.collect(null, new Text(badlen));
+      if (cbcopied != srcstat.getLen()) {
+        throw new IOException("File size not matched: copied "
+            + bytesString(cbcopied) + " to tmpfile (=" + tmpfile
+            + ") but expected " + bytesString(srcstat.getLen()) 
+            + " from " + srcstat.getPath());        
       }
       else {
         if (totfiles == 1) {
@@ -408,7 +406,16 @@ public class CopyFiles implements Tool {
           throw new IOException("Failed to craete parent dir: " + absdst.getParent());
         }
         rename(tmpfile, absdst);
-        updatePermissions(srcstat, destFileSys.getFileStatus(absdst));
+
+        FileStatus dststat = destFileSys.getFileStatus(absdst);
+        if (dststat.getLen() != srcstat.getLen()) {
+          destFileSys.delete(absdst, false);
+          throw new IOException("File size not matched: copied "
+              + bytesString(dststat.getLen()) + " to dst (=" + absdst
+              + ") but expected " + bytesString(srcstat.getLen()) 
+              + " from " + srcstat.getPath());        
+        } 
+        updatePermissions(srcstat, dststat);
       }
 
       // report at least once for each file