YARN-204. test coverage for org.apache.hadoop.tools (Aleksey Gorshkov via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1414239 13f79535-47bb-0310-9956-ffa450edef68

hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java

@@ -27,8 +27,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.StringTokenizer;
 
-import junit.framework.TestCase;
-
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
@@ -38,111 +36,117 @@ import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.util.JarFinder;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * test {@link HadoopArchives}
  */
-public class TestHadoopArchives extends TestCase {
+public class TestHadoopArchives {
 
-  public static final String HADOOP_ARCHIVES_JAR = JarFinder.getJar(HadoopArchives.class);
+  public static final String HADOOP_ARCHIVES_JAR = JarFinder
+      .getJar(HadoopArchives.class);
 
   {
-    ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
-        ).getLogger().setLevel(Level.ERROR);
-    ((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
-        ).getLogger().setLevel(Level.ERROR);
-    ((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
-        ).getLogger().setLevel(Level.ERROR);
+    ((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
+        .getLogger().setLevel(Level.ERROR);
+
   }
 
   private static final String inputDir = "input";
 
   private Path inputPath;
   private MiniDFSCluster dfscluster;
-  private MiniMRCluster mapred;
+
+  private Configuration conf;
   private FileSystem fs;
   private Path archivePath;
-  
-  static private Path createFile(Path dir, String filename, FileSystem fs
-      ) throws IOException {
+
+  static private Path createFile(Path dir, String filename, FileSystem fs)
+      throws IOException {
     final Path f = new Path(dir, filename);
-    final FSDataOutputStream out = fs.create(f); 
+    final FSDataOutputStream out = fs.create(f);
     out.write(filename.getBytes());
     out.close();
     return f;
   }
-  
-  protected void setUp() throws Exception {
-    super.setUp();
-    dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null);
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    conf.set(CapacitySchedulerConfiguration.PREFIX
+        + CapacitySchedulerConfiguration.ROOT + "."
+        + CapacitySchedulerConfiguration.QUEUES, "default");
+    conf.set(CapacitySchedulerConfiguration.PREFIX
+        + CapacitySchedulerConfiguration.ROOT + ".default."
+        + CapacitySchedulerConfiguration.CAPACITY, "100");
+    dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true)
+        .build();
+
     fs = dfscluster.getFileSystem();
-    mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
-    inputPath = new Path(fs.getHomeDirectory(), inputDir); 
+    inputPath = new Path(fs.getHomeDirectory(), inputDir);
     archivePath = new Path(fs.getHomeDirectory(), "archive");
     fs.mkdirs(inputPath);
     createFile(inputPath, "a", fs);
     createFile(inputPath, "b", fs);
     createFile(inputPath, "c", fs);
   }
-  
-  protected void tearDown() throws Exception {
+
+  @After
+  public void tearDown() throws Exception {
     try {
-      if (mapred != null) {
-        mapred.shutdown();
+      if (dfscluster != null) {
+        dfscluster.shutdown();
       }
       if (dfscluster != null) {
         dfscluster.shutdown();
       }
-    } catch(Exception e) {
+    } catch (Exception e) {
       System.err.println(e);
     }
-    super.tearDown();
   }
-  
-   
+
+  @Test
   public void testRelativePath() throws Exception {
     fs.delete(archivePath, true);
 
     final Path sub1 = new Path(inputPath, "dir1");
     fs.mkdirs(sub1);
     createFile(sub1, "a", fs);
-    final Configuration conf = mapred.createJobConf();
     final FsShell shell = new FsShell(conf);
 
     final List<String> originalPaths = lsr(shell, "input");
     System.out.println("originalPath: " + originalPaths);
     final URI uri = fs.getUri();
-    final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort()
+    final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
         + archivePath.toUri().getPath() + Path.SEPARATOR;
 
     {
       final String harName = "foo.har";
-      final String[] args = {
-          "-archiveName",
-          harName,
-          "-p",
-          "input",
-          "*",
-          "archive"
-      };
-      System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, HADOOP_ARCHIVES_JAR);
-      final HadoopArchives har = new HadoopArchives(mapred.createJobConf());
-      assertEquals(0, ToolRunner.run(har, args));
-
-      //compare results
+      final String[] args = { "-archiveName", harName, "-p", "input", "*",
+          "archive" };
+      System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
+          HADOOP_ARCHIVES_JAR);
+      final HadoopArchives har = new HadoopArchives(conf);
+      Assert.assertEquals(0, ToolRunner.run(har, args));
+
+      // compare results
       final List<String> harPaths = lsr(shell, prefix + harName);
-      assertEquals(originalPaths, harPaths);
+      Assert.assertEquals(originalPaths, harPaths);
     }
   }
-
+  
+@Test
   public void testPathWithSpaces() throws Exception {
     fs.delete(archivePath, true);
 
-    //create files/directories with spaces
+    // create files/directories with spaces
     createFile(inputPath, "c c", fs);
     final Path sub1 = new Path(inputPath, "sub 1");
     fs.mkdirs(sub1);
@@ -154,42 +158,36 @@ public class TestHadoopArchives extends TestCase {
     final Path sub2 = new Path(inputPath, "sub 1 with suffix");
     fs.mkdirs(sub2);
     createFile(sub2, "z", fs);
-    final Configuration conf = mapred.createJobConf();
+
     final FsShell shell = new FsShell(conf);
 
     final String inputPathStr = inputPath.toUri().getPath();
-    System.out.println("inputPathStr = " + inputPathStr);
 
     final List<String> originalPaths = lsr(shell, inputPathStr);
     final URI uri = fs.getUri();
-    final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort()
+    final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
         + archivePath.toUri().getPath() + Path.SEPARATOR;
 
-    {//Enable space replacement
+    {// Enable space replacement
       final String harName = "foo.har";
-      final String[] args = {
-          "-archiveName",
-          harName,
-          "-p",
-          inputPathStr,
-          "*",
-          archivePath.toString()
-      };
-      System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH, HADOOP_ARCHIVES_JAR);
-      final HadoopArchives har = new HadoopArchives(mapred.createJobConf());
-      assertEquals(0, ToolRunner.run(har, args));
-
-      //compare results
+      final String[] args = { "-archiveName", harName, "-p", inputPathStr, "*",
+          archivePath.toString() };
+      System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
+          HADOOP_ARCHIVES_JAR);
+      final HadoopArchives har = new HadoopArchives(conf);
+      Assert.assertEquals(0, ToolRunner.run(har, args));
+
+      // compare results
       final List<String> harPaths = lsr(shell, prefix + harName);
-      assertEquals(originalPaths, harPaths);
+      Assert.assertEquals(originalPaths, harPaths);
     }
 
   }
 
-  private static List<String> lsr(final FsShell shell, String dir
-      ) throws Exception {
+  private static List<String> lsr(final FsShell shell, String dir)
+      throws Exception {
     System.out.println("lsr root=" + dir);
-    final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
+    final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
@@ -197,7 +195,7 @@ public class TestHadoopArchives extends TestCase {
     System.setErr(out);
     final String results;
     try {
-      assertEquals(0, shell.run(new String[]{"-lsr", dir}));
+      Assert.assertEquals(0, shell.run(new String[] { "-lsr", dir }));
       results = bytes.toString();
     } finally {
       IOUtils.closeStream(out);
@@ -206,13 +204,13 @@ public class TestHadoopArchives extends TestCase {
     }
     System.out.println("lsr results:\n" + results);
     String dirname = dir;
-    if (dir.lastIndexOf(Path.SEPARATOR) != -1 ) {
+    if (dir.lastIndexOf(Path.SEPARATOR) != -1) {
       dirname = dir.substring(dir.lastIndexOf(Path.SEPARATOR));
     }
 
     final List<String> paths = new ArrayList<String>();
-    for(StringTokenizer t = new StringTokenizer(results, "\n");
-        t.hasMoreTokens(); ) {
+    for (StringTokenizer t = new StringTokenizer(results, "\n"); t
+        .hasMoreTokens();) {
       final String s = t.nextToken();
       final int i = s.indexOf(dirname);
       if (i >= 0) {
@@ -220,7 +218,8 @@ public class TestHadoopArchives extends TestCase {
       }
     }
     Collections.sort(paths);
-    System.out.println("lsr paths = " + paths.toString().replace(", ", ",\n  "));
+    System.out
+        .println("lsr paths = " + paths.toString().replace(", ", ",\n  "));
     return paths;
   }
 }

hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java

@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Cluster;
+import org.apache.hadoop.mapreduce.JobSubmissionFiles;
+import org.apache.hadoop.tools.util.TestDistCpUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.Permission;
+
+public class TestExternalCall {
+
+  private static final Log LOG = LogFactory.getLog(TestExternalCall.class);
+
+  private static FileSystem fs;
+
+  private static String root;
+
+  private static Configuration getConf() {
+    Configuration conf = new Configuration();
+    conf.set("fs.default.name", "file:///");
+    conf.set("mapred.job.tracker", "local");
+    return conf;
+  }
+
+  @Before
+  public void setup() {
+
+    securityManager = System.getSecurityManager();
+    System.setSecurityManager(new NoExitSecurityManager());
+    try {
+      fs = FileSystem.get(getConf());
+      root = new Path("target/tmp").makeQualified(fs.getUri(),
+          fs.getWorkingDirectory()).toString();
+      TestDistCpUtils.delete(fs, root);
+    } catch (IOException e) {
+      LOG.error("Exception encountered ", e);
+    }
+  }
+
+  @After
+  public void tearDown() {
+    System.setSecurityManager(securityManager);
+  }
+/**
+ * test methods run end execute of DistCp class. silple copy file
+ * @throws Exception 
+ */
+  @Test
+  public void testCleanup() throws Exception {
+
+      Configuration conf = getConf();
+
+      Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf),
+          conf);
+      stagingDir.getFileSystem(conf).mkdirs(stagingDir);
+      Path soure = createFile("tmp.txt");
+      Path target = createFile("target.txt");
+
+      DistCp distcp = new DistCp(conf, null);
+      String[] arg = { soure.toString(), target.toString() };
+
+      distcp.run(arg);
+      Assert.assertTrue(fs.exists(target));
+
+  
+  }
+
+  private Path createFile(String fname) throws IOException {
+    Path result = new Path(root + "/" + fname);
+    OutputStream out = fs.create(result);
+    try {
+      out.write((root + "/" + fname).getBytes());
+      out.write("\n".getBytes());
+    } finally {
+      out.close();
+    }
+    return result;
+  }
+
+  /**
+   * test main method of DistCp. Method should to call System.exit().
+   * 
+   */
+  @Test
+  public void testCleanupTestViaToolRunner() throws IOException, InterruptedException {
+
+    Configuration conf = getConf();
+
+    Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf);
+    stagingDir.getFileSystem(conf).mkdirs(stagingDir);
+   
+    Path soure = createFile("tmp.txt");
+    Path target = createFile("target.txt");
+    try {
+
+      String[] arg = {target.toString(),soure.toString()};
+      DistCp.main(arg);
+      Assert.fail();
+
+    } catch (ExitException t) {
+      Assert.assertTrue(fs.exists(target));
+      Assert.assertEquals(t.status, 0);
+      Assert.assertEquals(
+          stagingDir.getFileSystem(conf).listStatus(stagingDir).length, 0);
+    }
+
+  }
+
+  private SecurityManager securityManager;
+
+  protected static class ExitException extends SecurityException {
+    private static final long serialVersionUID = -1982617086752946683L;
+    public final int status;
+
+    public ExitException(int status) {
+      super("There is no escape!");
+      this.status = status;
+    }
+  }
+
+  private static class NoExitSecurityManager extends SecurityManager {
+    @Override
+    public void checkPermission(Permission perm) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkPermission(Permission perm, Object context) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkExit(int status) {
+      super.checkExit(status);
+      throw new ExitException(status);
+    }
+  }
+}

hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java

@@ -43,21 +43,19 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.Builder;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.tools.DistCpV1;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.log4j.Level;
-import org.junit.Ignore;
 
 
 /**
  * A JUnit test for copying files recursively.
  */
-@Ignore
+
 public class TestCopyFiles extends TestCase {
   {
     ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
@@ -738,20 +736,22 @@ public class TestCopyFiles extends TestCase {
   public void testMapCount() throws Exception {
     String namenode = null;
     MiniDFSCluster dfs = null;
-    MiniMRCluster mr = null;
+    MiniDFSCluster mr = null;
     try {
       Configuration conf = new Configuration();
-      dfs = new MiniDFSCluster(conf, 3, true, null);
+      
+      dfs= new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
+      
       FileSystem fs = dfs.getFileSystem();
       final FsShell shell = new FsShell(conf);
       namenode = fs.getUri().toString();
-      mr = new MiniMRCluster(3, namenode, 1);
       MyFile[] files = createFiles(fs.getUri(), "/srcdat");
       long totsize = 0;
       for (MyFile f : files) {
         totsize += f.getSize();
       }
-      Configuration job = mr.createJobConf();
+      
+      Configuration job = new JobConf(conf);
       job.setLong("distcp.bytes.per.map", totsize / 3);
       ToolRunner.run(new DistCpV1(job),
           new String[] {"-m", "100",
@@ -766,8 +766,7 @@ public class TestCopyFiles extends TestCase {
       System.out.println(execCmd(shell, "-lsr", logdir));
       FileStatus[] logs = fs.listStatus(new Path(logdir));
       // rare case where splits are exact, logs.length can be 4
-      assertTrue("Unexpected map count, logs.length=" + logs.length,
-          logs.length == 5 || logs.length == 4);
+      assertTrue( logs.length == 2);
 
       deldir(fs, "/destdat");
       deldir(fs, "/logs");

hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java

@@ -22,8 +22,6 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Random;
 
 import org.apache.commons.logging.LogFactory;
@@ -39,10 +37,10 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.log4j.Level;
-import org.junit.Ignore;
-@Ignore
+
 public class TestDistCh extends junit.framework.TestCase {
   {
     ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
@@ -52,7 +50,8 @@ public class TestDistCh extends junit.framework.TestCase {
   }
 
   static final Long RANDOM_NUMBER_GENERATOR_SEED = null;
-
+  static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+  
   private static final Random RANDOM = new Random();
   static {
     final long seed = RANDOM_NUMBER_GENERATOR_SEED == null?
@@ -65,7 +64,7 @@ public class TestDistCh extends junit.framework.TestCase {
     new Path(System.getProperty("test.build.data","/tmp")
         ).toString().replace(' ', '+');
 
-  static final int NUN_SUBS = 5;
+  static final int NUN_SUBS = 7;
 
   static class FileTree {
     private final FileSystem fs;
@@ -127,9 +126,12 @@ public class TestDistCh extends junit.framework.TestCase {
   
   public void testDistCh() throws Exception {
     final Configuration conf = new Configuration();
-    final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+
+    conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+"."+CapacitySchedulerConfiguration.QUEUES, "default");
+    conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+".default."+CapacitySchedulerConfiguration.CAPACITY, "100");
+    final MiniDFSCluster cluster=  new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build();
+    
     final FileSystem fs = cluster.getFileSystem();
-    final MiniMRCluster mr = new MiniMRCluster(2, fs.getUri().toString(), 1);
     final FsShell shell = new FsShell(conf);
     
     try {
@@ -138,37 +140,36 @@ public class TestDistCh extends junit.framework.TestCase {
 
       runLsr(shell, tree.root, 0);
 
-      //generate random arguments
-      final String[] args = new String[RANDOM.nextInt(NUN_SUBS-1) + 1];
+      final String[] args = new String[NUN_SUBS];
       final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
-      final List<Integer> indices = new LinkedList<Integer>();
-      for(int i = 0; i < NUN_SUBS; i++) {
-        indices.add(i);
-      }
-      for(int i = 0; i < args.length; i++) {
-        final int index = indices.remove(RANDOM.nextInt(indices.size()));
-        final String sub = "sub" + index;
-        final boolean changeOwner = RANDOM.nextBoolean();
-        final boolean changeGroup = RANDOM.nextBoolean();
-        final boolean changeMode = !changeOwner && !changeGroup? true: RANDOM.nextBoolean();
-        
-        final String owner = changeOwner? sub: "";
-        final String group = changeGroup? sub: "";
-        final String permission = changeMode? RANDOM.nextInt(8) + "" + RANDOM.nextInt(8) + "" + RANDOM.nextInt(8): "";
-
-        args[i] = tree.root + "/" + sub + ":" + owner + ":" + group + ":" + permission;
-        newstatus[index] = new ChPermissionStatus(rootstatus, owner, group, permission);
-      }
-      for(int i = 0; i < NUN_SUBS; i++) {
-        if (newstatus[i] == null) {
-          newstatus[i] = new ChPermissionStatus(rootstatus);
-        }
-      }
+
+      
+      args[0]="/test/testDistCh/sub0:sub1::";
+      newstatus[0] = new ChPermissionStatus(rootstatus, "sub1", "", "");
+
+      args[1]="/test/testDistCh/sub1::sub2:";
+      newstatus[1] = new ChPermissionStatus(rootstatus, "", "sub2", "");
+
+      args[2]="/test/testDistCh/sub2:::437";
+      newstatus[2] = new ChPermissionStatus(rootstatus, "", "", "437");
+
+      args[3]="/test/testDistCh/sub3:sub1:sub2:447";
+      newstatus[3] = new ChPermissionStatus(rootstatus, "sub1", "sub2", "447");
+ 
+      args[4]="/test/testDistCh/sub4::sub5:437";
+      newstatus[4] = new ChPermissionStatus(rootstatus, "", "sub5", "437");
+
+      args[5]="/test/testDistCh/sub5:sub1:sub5:";
+      newstatus[5] = new ChPermissionStatus(rootstatus, "sub1", "sub5", "");
+
+      args[6]="/test/testDistCh/sub6:sub3::437";
+      newstatus[6] = new ChPermissionStatus(rootstatus, "sub3", "", "437");
+      
       System.out.println("args=" + Arrays.asList(args).toString().replace(",", ",\n  "));
       System.out.println("newstatus=" + Arrays.asList(newstatus).toString().replace(",", ",\n  "));
 
       //run DistCh
-      new DistCh(mr.createJobConf()).run(args);
+      new DistCh(MiniMRClientClusterFactory.create(this.getClass(), 2, conf).getConfig()).run(args);
       runLsr(shell, tree.root, 0);
 
       //check results
@@ -184,7 +185,7 @@ public class TestDistCh extends junit.framework.TestCase {
     }
   }
 
-  static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+ 
 
   static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
     assertEquals(expected.getUserName(), actual.getOwner());

hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.EnumSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestLogalyzer {
+  private static String EL = System.getProperty("line.separator");
+  private static String TAB = "\t";
+  private static final Log LOG = LogFactory.getLog(TestLogalyzer.class);
+
+  private static File workSpace = new File("target",
+      TestLogalyzer.class.getName() + "-workSpace");
+  private static File outdir = new File(workSpace.getAbsoluteFile()
+      + File.separator + "out");
+
+  @Test
+  public void testLogalyzer() throws Exception {
+    Path f = createLogFile();
+
+    String[] args = new String[10];
+
+    args[0] = "-archiveDir";
+    args[1] = f.toString();
+    args[2] = "-grep";
+    args[3] = "44";
+    args[4] = "-sort";
+    args[5] = "0";
+    args[6] = "-analysis";
+    args[7] = outdir.getAbsolutePath();
+    args[8] = "-separator";
+    args[9] = " ";
+
+    Logalyzer.main(args);
+    checkResult();
+
+  }
+
+  private void checkResult() throws Exception {
+    File result = new File(outdir.getAbsolutePath() + File.separator
+        + "part-00000");
+    File success = new File(outdir.getAbsolutePath() + File.separator
+        + "_SUCCESS");
+    Assert.assertTrue(success.exists());
+
+    FileInputStream fis = new FileInputStream(result);
+    BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
+    String line = br.readLine();
+    Assert.assertTrue(("1 44" + TAB + "2").equals(line));
+    line = br.readLine();
+
+    Assert.assertTrue(("3 44" + TAB + "1").equals(line));
+    line = br.readLine();
+
+    Assert.assertTrue(("4 44" + TAB + "1").equals(line));
+
+    br.close();
+
+  }
+
+  /**
+   * Create simple log file
+   * 
+   * @return
+   * @throws IOException
+   */
+
+  private Path createLogFile() throws IOException {
+
+    FileContext files = FileContext.getLocalFSFileContext();
+
+    Path ws = new Path(workSpace.getAbsoluteFile().getAbsolutePath());
+
+    files.delete(ws, true);
+    Path workSpacePath = new Path(workSpace.getAbsolutePath(), "log");
+    files.mkdir(workSpacePath, null, true);
+
+    LOG.info("create logfile.log");
+    Path logfile1 = new Path(workSpacePath, "logfile.log");
+
+    FSDataOutputStream os = files.create(logfile1,
+        EnumSet.of(CreateFlag.CREATE));
+    os.writeBytes("4 3" + EL + "1 3" + EL + "4 44" + EL);
+    os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
+    os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);
+
+    os.flush();
+    os.close();
+    LOG.info("create logfile1.log");
+
+    Path logfile2 = new Path(workSpacePath, "logfile1.log");
+
+    os = files.create(logfile2, EnumSet.of(CreateFlag.CREATE));
+    os.writeBytes("4 3" + EL + "1 3" + EL + "3 44" + EL);
+    os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
+    os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);
+
+    os.flush();
+    os.close();
+
+    return workSpacePath;
+  }
+}

hadoop-yarn-project/CHANGES.txt

@@ -180,6 +180,9 @@ Release 0.23.6 - UNRELEASED
     YARN-151. Browser thinks RM main page JS is taking too long 
     (Ravi Prakash via bobby)
 
+    YARN-204. test coverage for org.apache.hadoop.tools (Aleksey Gorshkov via
+    bobby)
+
 Release 0.23.5 - UNRELEASED
 
   INCOMPATIBLE CHANGES