HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to branch-trunk-win to enable initial test pass. Contributed by Raja Aluri, Davio Lao, Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, Ivan Mitic, Chris Nauroth, and Bikas Saha.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-trunk-win@1401443 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt

@@ -9,3 +9,8 @@ branch-trunk-win changes - unreleased
 
   HADOOP-8946. winutils: compile codebase during Maven build on 
   branch-trunk-win. (Chris Nauroth via suresh)
+
+  HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to 
+  branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, 
+  Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, 
+  Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java

@@ -143,11 +143,23 @@ public class DF extends Shell {
       mount;
   }
 
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      try {
+        this.mount = dirFile.getCanonicalPath().substring(0,2);
+      } catch (IOException e) {
+      }
+      return;
+    }
+    super.run();
+  }
+
   @Override
   protected String[] getExecString() {
     // ignoring the error since the exit code it enough
-    return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath 
-                         + "' 2>/dev/null"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "df -k " + dirPath + " 2>nul"}:
+        new String[] {"bash","-c","exec 'df' '-k' '" + dirPath + "' 2>/dev/null"};
   }
 
   @Override

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java

@@ -145,6 +145,20 @@ public class DU extends Shell {
   public String getDirPath() {
     return dirPath;
   }
+
+
+  /**
+   * Override to hook in DUHelper class. Maybe this can be used more
+   * generally as well on Unix/Linux based systems
+   */
+  @Override
+  protected void run() throws IOException {
+    if (WINDOWS) {
+      used.set(DUHelper.getFolderUsage(dirPath));
+      return;
+    }
+    super.run();
+  }
   
   /**
    * Start the disk usage checking thread.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java

@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.File;
+import org.apache.hadoop.util.Shell;
+
+public class DUHelper {
+
+  private int folderCount=0;
+  private int fileCount=0;
+  private double usage = 0;
+  private long folderSize = -1;
+
+  private DUHelper() {
+
+  }
+
+  public static long getFolderUsage(String folder) {
+    return new DUHelper().calculateFolderSize(folder);
+  }
+
+  private long calculateFolderSize(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+    return folderSize = getFileSize(f);
+  }
+
+  public String check(String folder) {
+    if (folder == null)
+      throw new IllegalArgumentException("folder");
+    File f = new File(folder);
+
+    folderSize = getFileSize(f);
+    usage = 1.0*(f.getTotalSpace() - f.getFreeSpace())/ f.getTotalSpace();
+    return String.format("used %d files %d disk in use %f", folderSize, fileCount, usage);
+  }
+
+  public long getFileCount() {
+    return fileCount;
+  }
+
+  public double getUsage() {
+    return usage;
+  }
+
+  private long getFileSize(File folder) {
+
+    folderCount++;
+    //Counting the total folders
+    long foldersize = 0;
+    if (folder.isFile())
+      return folder.length();
+    File[] filelist = folder.listFiles();
+    if (filelist == null) {
+      return 0;
+    }
+    for (int i = 0; i < filelist.length; i++) {
+      if (filelist[i].isDirectory()) {
+        foldersize += getFileSize(filelist[i]);
+      } else {
+        fileCount++; //Counting the total files
+        foldersize += filelist[i].length();
+      }
+    }
+    return foldersize;    
+  }
+
+  public static void main(String[] args) {
+    if (Shell.WINDOWS)
+      System.out.println("Windows: "+ DUHelper.getFolderUsage(args[0]));
+    else
+      System.out.println("Other: " + DUHelper.getFolderUsage(args[0]));
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java

@@ -27,7 +27,11 @@ import java.util.zip.ZipFile;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
@@ -43,6 +47,13 @@ public class FileUtil {
 
   private static final Log LOG = LogFactory.getLog(FileUtil.class);
 
+  /* The error code is defined in winutils to indicate insufficient
+   * privilege to create symbolic links. This value need to keep in
+   * sync with the constant of the same name in:
+   * "src\winutils\common.h"
+   * */
+  public static final int SYMLINK_NO_PRIVILEGE = 2;
+
   /**
    * convert an array of FileStatus to an array of Path
    * 
@@ -436,11 +447,7 @@ public class FileUtil {
    * @throws IOException on windows, there can be problems with the subprocess
    */
   public static String makeShellPath(String filename) throws IOException {
-    if (Path.WINDOWS) {
-      return new CygPathCommand(filename).getResult();
-    } else {
-      return filename;
-    }    
+    return filename;
   }
   
   /**
@@ -568,21 +575,24 @@ public class FileUtil {
     StringBuilder untarCommand = new StringBuilder();
     boolean gzipped = inFile.toString().endsWith("gz");
     if (gzipped) {
-      untarCommand.append(" gzip -dc '");
+      untarCommand.append((Shell.WINDOWS) ? " gzip -dc \"" : " gzip -dc '");
       untarCommand.append(FileUtil.makeShellPath(inFile));
-      untarCommand.append("' | (");
+      untarCommand.append((Shell.WINDOWS) ? "\" | (" : "' | (");
     } 
-    untarCommand.append("cd '");
+    untarCommand.append((Shell.WINDOWS) ? "cd \"" : "cd '");
     untarCommand.append(FileUtil.makeShellPath(untarDir)); 
-    untarCommand.append("' ; ");
-    untarCommand.append("tar -xf ");
-    
+    untarCommand.append((Shell.WINDOWS) ? "\" & " : "' ; ");
+
+    // Force the archive path as local on Windows as it can have a colon
+    untarCommand.append((Shell.WINDOWS) ? "tar --force-local -xf " : "tar -xf ");
+
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
       untarCommand.append(FileUtil.makeShellPath(inFile));
     }
-    String[] shellCmd = { "bash", "-c", untarCommand.toString() };
+    String[] shellCmd = {(Shell.WINDOWS)?"cmd":"bash", (Shell.WINDOWS)?"/c":"-c",
+      untarCommand.toString() };
     ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
     shexec.execute();
     int exitcode = shexec.getExitCode();
@@ -606,21 +616,44 @@ public class FileUtil {
 
   /**
    * Create a soft link between a src and destination
-   * only on a local disk. HDFS does not support this
+   * only on a local disk. HDFS does not support this.
+   * On Windows, when symlink creation fails due to security
+   * setting, we will log a warning. The return code in this
+   * case is 2.
    * @param target the target for symlink 
    * @param linkname the symlink
    * @return value returned by the command
    */
   public static int symLink(String target, String linkname) throws IOException{
-    String cmd = "ln -s " + target + " " + linkname;
-    Process p = Runtime.getRuntime().exec(cmd, null);
-    int returnVal = -1;
-    try{
-      returnVal = p.waitFor();
-    } catch(InterruptedException e){
-      //do nothing as of yet
+    // Run the input paths through Java's File so that they are converted to the
+    // native OS form. FIXME: Long term fix is to expose symLink API that
+    // accepts File instead of String, as symlinks can only be created on the
+    // local FS.
+    String[] cmd = Shell.getSymlinkCommand(new File(target).getPath(),
+        new File(linkname).getPath());
+    ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+    try {
+      shExec.execute();
+    } catch (Shell.ExitCodeException ec) {
+      int returnVal = ec.getExitCode();
+      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+        LOG.warn("Fail to create symbolic links on Windows. "
+            + "The default security settings in Windows disallow non-elevated "
+            + "administrators and all non-administrators from creating symbolic links. "
+            + "This behavior can be changed in the Local Security Policy management console");
+      } else if (returnVal != 0) {
+        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+            + returnVal + " with: " + ec.getMessage());
+      }
+      return returnVal;
+    } catch (IOException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Error while create symlink " + linkname + " to " + target
+            + "." + " Exception: " + StringUtils.stringifyException(e));
+      }
+      throw e;
     }
-    return returnVal;
+    return shExec.getExitCode();
   }
   
   /**
@@ -644,30 +677,101 @@ public class FileUtil {
    * @param recursive true, if permissions should be changed recursively
    * @return the exit code from the command.
    * @throws IOException
-   * @throws InterruptedException
    */
   public static int chmod(String filename, String perm, boolean recursive)
-                            throws IOException, InterruptedException {
-    StringBuilder cmdBuf = new StringBuilder();
-    cmdBuf.append("chmod ");
-    if (recursive) {
-      cmdBuf.append("-R ");
-    }
-    cmdBuf.append(perm).append(" ");
-    cmdBuf.append(filename);
-    String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
-    ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+                            throws IOException {
+    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = new File(filename).getPath();
+    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
     try {
       shExec.execute();
-    }catch(Exception e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Error while changing permission : " + filename
-            + " Exception: ", e);
+    }catch(IOException e) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Error while changing permission : " + filename 
+                  +" Exception: " + StringUtils.stringifyException(e));
       }
     }
     return shExec.getExitCode();
   }
+
+  /**
+   * Set permissions to the required value. Uses the java primitives instead
+   * of forking if group == other.
+   * @param f the file to change
+   * @param permission the new permissions
+   * @throws IOException
+   */
+  public static void setPermission(File f, FsPermission permission
+                                   ) throws IOException {
+    FsAction user = permission.getUserAction();
+    FsAction group = permission.getGroupAction();
+    FsAction other = permission.getOtherAction();
+
+    // use the native/fork if the group/other permissions are different
+    // or if the native is available or on Windows
+    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+      execSetPermission(f, permission);
+      return;
+    }
+    
+    boolean rv = true;
+    
+    // read perms
+    rv = f.setReadable(group.implies(FsAction.READ), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+      f.setReadable(user.implies(FsAction.READ), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // write perms
+    rv = f.setWritable(group.implies(FsAction.WRITE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+      f.setWritable(user.implies(FsAction.WRITE), true);
+      checkReturnValue(rv, f, permission);
+    }
+
+    // exec perms
+    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+    checkReturnValue(rv, f, permission);
+    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+      f.setExecutable(user.implies(FsAction.EXECUTE), true);
+      checkReturnValue(rv, f, permission);
+    }
+  }
+
+  private static void checkReturnValue(boolean rv, File p, 
+                                       FsPermission permission
+                                       ) throws IOException {
+    if (!rv) {
+      throw new IOException("Failed to set permissions of path: " + p + 
+                            " to " + 
+                            String.format("%04o", permission.toShort()));
+    }
+  }
   
+  private static void execSetPermission(File f, 
+                                        FsPermission permission
+                                       )  throws IOException {
+    if (NativeIO.isAvailable()) {
+      NativeIO.chmod(f.getCanonicalPath(), permission.toShort());
+    } else {
+      execCommand(f, Shell.getSetPermissionCommand(
+                  String.format("%04o", permission.toShort()), false));
+    }
+  }
+  
+  static String execCommand(File f, String... cmd) throws IOException {
+    String[] args = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, args, 0, cmd.length);
+    args[cmd.length] = f.getCanonicalPath();
+    String output = Shell.execCommand(args);
+    return output;
+  }
+
   /**
    * Create a tmp file for a base file.
    * @param basefile the base file of the tmp

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -36,7 +36,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
@@ -529,8 +528,8 @@ public class RawLocalFileSystem extends FileSystem {
       IOException e = null;
       try {
         StringTokenizer t = new StringTokenizer(
-            execCommand(new File(getPath().toUri()), 
-                        Shell.getGET_PERMISSION_COMMAND()));
+            FileUtil.execCommand(new File(getPath().toUri()), 
+                                          Shell.getGetPermissionCommand()));
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
@@ -539,7 +538,18 @@ public class RawLocalFileSystem extends FileSystem {
         }
         setPermission(FsPermission.valueOf(permission));
         t.nextToken();
-        setOwner(t.nextToken());
+
+        String owner = t.nextToken();
+        // If on windows domain, token format is DOMAIN\\user and we want to
+        // extract only the user name
+        if (Shell.WINDOWS) {
+          int i = owner.indexOf('\\');
+          if (i != -1)
+            owner = owner.substring(i + 1);
+        }
+        setOwner(owner);
+
+        // FIXME: Group names could have spaces on Windows
         setGroup(t.nextToken());
       } catch (Shell.ExitCodeException ioe) {
         if (ioe.getExitCode() != 1) {
@@ -580,11 +590,11 @@ public class RawLocalFileSystem extends FileSystem {
     }
 
     if (username == null) {
-      execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
+      FileUtil.execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
     } else {
       //OWNER[:[GROUP]]
       String s = username + (groupname == null? "": ":" + groupname);
-      execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
+      FileUtil.execCommand(pathToFile(p), Shell.getSetOwnerCommand(s));
     }
   }
 
@@ -592,23 +602,7 @@ public class RawLocalFileSystem extends FileSystem {
    * Use the command chmod to set permission.
    */
   @Override
-  public void setPermission(Path p, FsPermission permission)
-    throws IOException {
-    if (NativeIO.isAvailable()) {
-      NativeIO.chmod(pathToFile(p).getCanonicalPath(),
-                     permission.toShort());
-    } else {
-      execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
-          String.format("%05o", permission.toShort()));
-    }
-  }
-
-  private static String execCommand(File f, String... cmd) throws IOException {
-    String[] args = new String[cmd.length + 1];
-    System.arraycopy(cmd, 0, args, 0, cmd.length);
-    args[cmd.length] = FileUtil.makeShellPath(f, true);
-    String output = Shell.execCommand(args);
-    return output;
+  public void setPermission(Path p, FsPermission permission) throws IOException {
+    FileUtil.setPermission(pathToFile(p), permission);
   }
-
-}
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

@@ -44,36 +44,76 @@ abstract public class Shell {
   
   public static final Log LOG = LogFactory.getLog(Shell.class);
   
+  /** a Windows utility to emulate Unix commands */
+  public static final String WINUTILS = System.getenv("HADOOP_HOME")
+                                        + "\\bin\\winutils";
+
   /** a Unix command to get the current user's name */
   public final static String USER_NAME_COMMAND = "whoami";
+
+  /** Windows CreateProcess synchronization object */
+  public static final Object WindowsProcessLaunchLock = new Object();
+
   /** a Unix command to get the current user's groups list */
   public static String[] getGroupsCommand() {
-    return new String[]{"bash", "-c", "groups"};
+    return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
+                    : new String[]{"bash", "-c", "groups"};
   }
+
   /** a Unix command to get a given user's groups list */
   public static String[] getGroupsForUserCommand(final String user) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "id -Gn " + user};
+    return (WINDOWS)? new String[] { WINUTILS, "groups", user}
+                    : new String [] {"bash", "-c", "id -Gn " + user};
   }
+
   /** a Unix command to get a given netgroup's user list */
   public static String[] getUsersForNetgroupCommand(final String netgroup) {
     //'groups username' command return is non-consistent across different unixes
-    return new String [] {"bash", "-c", "getent netgroup " + netgroup};
+    return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup}
+                    : new String [] {"bash", "-c", "getent netgroup " + netgroup};
+  }
+
+  /** Return a command to get permission information. */
+  public static String[] getGetPermissionCommand() {
+    return (WINDOWS) ? new String[] { WINUTILS, "ls" }
+                     : new String[] { "/bin/ls", "-ld" };
+  }
+
+  /** Return a command to set permission */
+  public static String[] getSetPermissionCommand(String perm, boolean recursive) {
+    if (recursive) {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm }
+                         : new String[] { "chmod", "-R", perm };
+    } else {
+      return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm }
+                       : new String[] { "chmod", perm };
+    }
   }
+
+  /** Return a command to set owner */
+  public static String[] getSetOwnerCommand(String owner) {
+    return (WINDOWS) ? new String[] { WINUTILS, "chown", owner }
+                     : new String[] { "chown", owner };
+  }
+
+  /** Return a command to create symbolic links */
+  public static String[] getSymlinkCommand(String target, String link) {
+    return WINDOWS ? new String[] { WINUTILS, "symlink", link, target }
+                   : new String[] { "ln", "-s", target, link };
+  }
+
   /** a Unix command to set permission */
   public static final String SET_PERMISSION_COMMAND = "chmod";
   /** a Unix command to set owner */
   public static final String SET_OWNER_COMMAND = "chown";
+
+  /** a Unix command to set the change user's groups list */
   public static final String SET_GROUP_COMMAND = "chgrp";
   /** a Unix command to create a link */
   public static final String LINK_COMMAND = "ln";
   /** a Unix command to get a link target */
   public static final String READ_LINK_COMMAND = "readlink";
-  /** Return a Unix command to get permission information. */
-  public static String[] getGET_PERMISSION_COMMAND() {
-    //force /bin/ls, except on windows.
-    return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"};
-  }
 
   /**Time after which the executing script would be timedout*/
   protected long timeOutInterval = 0L;
@@ -144,7 +184,19 @@ abstract public class Shell {
       builder.directory(this.dir);
     }
     
-    process = builder.start();
+    if (Shell.WINDOWS) {
+      synchronized (WindowsProcessLaunchLock) {
+        // To workaround the race condition issue with child processes
+        // inheriting unintended handles during process launch that can
+        // lead to hangs on reading output and error streams, we
+        // serialize process creation. More info available at:
+        // http://support.microsoft.com/kb/315939
+        process = builder.start();
+      }
+    } else {
+      process = builder.start();
+    }
+
     if (timeOutInterval > 0) {
       timeOutTimer = new Timer("Shell command timeout");
       timeoutTimerTask = new ShellTimeoutTimerTask(

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -778,6 +778,28 @@ public class StringUtils {
     return sb.toString();
   }
 
+  /**
+   * Concatenates strings, using a separator.
+   *
+   * @param separator to join with
+   * @param strings to join
+   * @return  the joined string
+   */
+  public static String join(CharSequence separator, String[] strings) {
+    // Ideally we don't have to duplicate the code here if array is iterable.
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (String s : strings) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(separator);
+      }
+      sb.append(s);
+    }
+    return sb.toString();
+  }
+
   /**
    * Convert SOME_STUFF to SomeStuff
    *

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java

@@ -19,6 +19,8 @@ package org.apache.hadoop.fs;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -29,6 +31,7 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Test;
@@ -336,6 +339,10 @@ public class TestFileUtil {
 
   @Test
   public void testFailFullyDelete() throws IOException {
+    if(Shell.WINDOWS) {
+      // windows Dir.setWritable(false) does not work for directories
+      return;
+    }
     LOG.info("Running test to verify failure of fullyDelete()");
     setupDirsAndNonWritablePermissions();
     boolean ret = FileUtil.fullyDelete(new MyFile(del));
@@ -402,6 +409,10 @@ public class TestFileUtil {
 
   @Test
   public void testFailFullyDeleteContents() throws IOException {
+    if(Shell.WINDOWS) {
+      // windows Dir.setWritable(false) does not work for directories
+      return;
+    }
     LOG.info("Running test to verify failure of fullyDeleteContents()");
     setupDirsAndNonWritablePermissions();
     boolean ret = FileUtil.fullyDeleteContents(new MyFile(del));
@@ -477,4 +488,36 @@ public class TestFileUtil {
     long expected = 2 * (3 + System.getProperty("line.separator").length());
     Assert.assertEquals(expected, du);
   }
+
+  @Test
+  public void testSymlink() throws Exception {
+    Assert.assertFalse(del.exists());
+    del.mkdirs();
+
+    byte[] data = "testSymLink".getBytes();
+
+    File file = new File(del, FILE);
+    File link = new File(del, "_link");
+
+    //write some data to the file
+    FileOutputStream os = new FileOutputStream(file);
+    os.write(data);
+    os.close();
+
+    //create the symlink
+    FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+    //ensure that symlink length is correctly reported by Java
+    Assert.assertEquals(data.length, file.length());
+    Assert.assertEquals(Shell.WINDOWS ? 0 : data.length, link.length());
+
+    //ensure that we can read from link.
+    FileInputStream in = new FileInputStream(link);
+    long len = 0;
+    while (in.read() > 0) {
+      len++;
+    }
+    in.close();
+    Assert.assertEquals(data.length, len);
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java

@@ -81,6 +81,10 @@ public class TestShell extends TestCase {
   }
   
   public void testShellCommandTimeout() throws Throwable {
+    if(Shell.WINDOWS) {
+      // setExecutable does not work on Windows
+      return;
+    }
     String rootDir = new File(System.getProperty(
         "test.build.data", "/tmp")).getAbsolutePath();
     File shellFile = new File(rootDir, "timeout.sh");