HADOOP-6963. In FileUtil.getDU(..), neither include the size of directories nor follow symbolic links. Contributed by Ravi Prakash

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1310039 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -233,6 +233,9 @@ Release 1.0.3 - unreleased
     HADOOP-5528. Ensure BinaryPartitioner is present in mapred libs. (Klaas
     Bosteels via acmurthy)
 
+    HADOOP-6963. In FileUtil.getDU(..), neither include the size of directories
+    nor follow symbolic links.  (Ravi Prakash via szetszwo)
+
 Release 1.0.2 - 2012.03.24
 
   NEW FEATURES

ivy.xml

@@ -85,6 +85,11 @@
       rev="${commons-cli.version}"
       conf="client->default"/>
 
+    <dependency org="commons-io"
+      name="commons-io"
+      rev="${commons-io.version}"
+      conf="client->default"/>
+
     <dependency org="checkstyle"
       name="checkstyle"
       rev="${checkstyle.version}"

ivy/libraries.properties

@@ -39,7 +39,7 @@ commons-logging-api.version=1.0.4
 commons-math.version=2.1
 commons-el.version=1.0
 commons-fileupload.version=1.2
-commons-io.version=1.4
+commons-io.version=2.1
 commons-net.version=3.1
 core.version=3.1.1
 coreplugin.version=1.3.2

src/core/org/apache/hadoop/fs/FileUtil.java

@@ -448,11 +448,18 @@ public class FileUtil {
     if (!dir.isDirectory()) {
       return dir.length();
     } else {
-      size = dir.length();
       File[] allFiles = dir.listFiles();
       if(allFiles != null) {
         for (int i = 0; i < allFiles.length; i++) {
-           size = size + getDU(allFiles[i]);
+          boolean isSymLink;
+          try {
+            isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
+          } catch(IOException ioe) {
+            isSymLink = true;
+          }
+          if(!isSymLink) {
+            size += getDU(allFiles[i]);
+          }
         }
       }
       return size;

src/test/org/apache/hadoop/fs/TestFileUtil.java

@@ -78,6 +78,9 @@ public class TestFileUtil {
     File linkDir = new File(del, "tmpDir");
     FileUtil.symLink(tmp.toString(), linkDir.toString());
     Assert.assertEquals(5, del.listFiles().length);
+
+    // create a cycle using symlinks. Cycles should be handled
+    FileUtil.symLink(del.toString(), del.toString() + "/" + DIR + "1/cycle");
   }
 
   @After
@@ -303,4 +306,17 @@ public class TestFileUtil {
       //Expected an IOException
     }
   }
+
+  /**
+   * Test that getDU is able to handle cycles caused due to symbolic links
+   * and that directory sizes are not added to the final calculated size
+   * @throws IOException
+   */
+  @Test
+  public void testGetDU() throws IOException {
+    setupDirs();
+
+    long du = FileUtil.getDU(TEST_DIR);
+    Assert.assertEquals(du, 0);
+  }
 }