HADOOP-3294. Fix distcp to check the destination length and retry the copy if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via mukund)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.16@652202 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -7,6 +7,9 @@ Release 0.16.4 - 2008-05-05
     HADOOP-3138. DFS mkdirs() should not throw an exception if the directory
     already exists. (rangadi via mukund)
 
+    HADOOP-3294. Fix distcp to check the destination length and retry the copy
+    if it doesn't match the src length. (Tsz Wo (Nicholas), SZE via mukund)
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

src/java/org/apache/hadoop/util/CopyFiles.java

@@ -319,13 +319,11 @@ public class CopyFiles implements Tool {
         checkAndClose(out);
       }
 
-      final boolean success = cbcopied == srcstat.getLen();
-      if (!success) {
-        final String badlen = "ERROR? copied " + bytesString(cbcopied)
-            + " but expected " + bytesString(srcstat.getLen()) 
-            + " from " + srcstat.getPath();
-        LOG.warn(badlen);
-        outc.collect(null, new Text(badlen));
+      if (cbcopied != srcstat.getLen()) {
+        throw new IOException("File size not matched: copied "
+            + bytesString(cbcopied) + " to tmpfile (=" + tmpfile
+            + ") but expected " + bytesString(srcstat.getLen()) 
+            + " from " + srcstat.getPath());        
       }
       else {
         if (totfiles == 1) {
@@ -345,6 +343,15 @@ public class CopyFiles implements Tool {
           throw new IOException("Failed to craete parent dir: " + absdst.getParent());
         }
         rename(destFileSys, tmpfile, absdst);
+
+        FileStatus dststat = destFileSys.getFileStatus(absdst);
+        if (dststat.getLen() != srcstat.getLen()) {
+          destFileSys.delete(absdst);
+          throw new IOException("File size not matched: copied "
+              + bytesString(dststat.getLen()) + " to dst (=" + absdst
+              + ") but expected " + bytesString(srcstat.getLen()) 
+              + " from " + srcstat.getPath());        
+        } 
       }
 
       // report at least once for each file