move test and xml files

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/HADOOP-4687/hdfs@779196 13f79535-47bb-0310-9956-ffa450edef68

src/test/hdfs-site.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+
+</configuration>

src/test/hdfs-with-mr/org/apache/hadoop/fs/AccumulatingReducer.java

@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Reducer that accumulates values based on their type.
+ * <p>
+ * The type is specified in the key part of the key-value pair 
+ * as a prefix to the key in the following way
+ * <p>
+ * <tt>type:key</tt>
+ * <p>
+ * The values are accumulated according to the types:
+ * <ul>
+ * <li><tt>s:</tt> - string, concatenate</li>
+ * <li><tt>f:</tt> - float, summ</li>
+ * <li><tt>l:</tt> - long, summ</li>
+ * </ul>
+ * 
+ */
+public class AccumulatingReducer extends MapReduceBase
+    implements Reducer<Text, Text, Text, Text> {
+  static final String VALUE_TYPE_LONG = "l:";
+  static final String VALUE_TYPE_FLOAT = "f:";
+  static final String VALUE_TYPE_STRING = "s:";
+  private static final Log LOG = LogFactory.getLog(AccumulatingReducer.class);
+  
+  protected String hostName;
+  
+  public AccumulatingReducer () {
+    LOG.info("Starting AccumulatingReducer !!!");
+    try {
+      hostName = java.net.InetAddress.getLocalHost().getHostName();
+    } catch(Exception e) {
+      hostName = "localhost";
+    }
+    LOG.info("Starting AccumulatingReducer on " + hostName);
+  }
+  
+  public void reduce(Text key, 
+                     Iterator<Text> values,
+                     OutputCollector<Text, Text> output, 
+                     Reporter reporter
+                     ) throws IOException {
+    String field = key.toString();
+
+    reporter.setStatus("starting " + field + " ::host = " + hostName);
+
+    // concatenate strings
+    if (field.startsWith(VALUE_TYPE_STRING)) {
+      String sSum = "";
+      while (values.hasNext())
+        sSum += values.next().toString() + ";";
+      output.collect(key, new Text(sSum));
+      reporter.setStatus("finished " + field + " ::host = " + hostName);
+      return;
+    }
+    // sum long values
+    if (field.startsWith(VALUE_TYPE_FLOAT)) {
+      float fSum = 0;
+      while (values.hasNext())
+        fSum += Float.parseFloat(values.next().toString());
+      output.collect(key, new Text(String.valueOf(fSum)));
+      reporter.setStatus("finished " + field + " ::host = " + hostName);
+      return;
+    }
+    // sum long values
+    if (field.startsWith(VALUE_TYPE_LONG)) {
+      long lSum = 0;
+      while (values.hasNext()) {
+        lSum += Long.parseLong(values.next().toString());
+      }
+      output.collect(key, new Text(String.valueOf(lSum)));
+    }
+    reporter.setStatus("finished " + field + " ::host = " + hostName);
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/DFSCIOTest.java

@@ -0,0 +1,551 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.Date;
+import java.util.StringTokenizer;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.mapred.*;
+
+/**
+ * Distributed i/o benchmark.
+ * <p>
+ * This test writes into or reads from a specified number of files.
+ * File size is specified as a parameter to the test. 
+ * Each file is accessed in a separate map task.
+ * <p>
+ * The reducer collects the following statistics:
+ * <ul>
+ * <li>number of tasks completed</li>
+ * <li>number of bytes written/read</li>
+ * <li>execution time</li>
+ * <li>io rate</li>
+ * <li>io rate squared</li>
+ * </ul>
+ *    
+ * Finally, the following information is appended to a local file
+ * <ul>
+ * <li>read or write test</li>
+ * <li>date and time the test finished</li>   
+ * <li>number of files</li>
+ * <li>total number of bytes processed</li>
+ * <li>throughput in mb/sec (total number of bytes / sum of processing times)</li>
+ * <li>average i/o rate in mb/sec per file</li>
+ * <li>standard i/o rate deviation</li>
+ * </ul>
+ */
+public class DFSCIOTest extends TestCase {
+  // Constants
+  private static final Log LOG = LogFactory.getLog(DFSCIOTest.class);
+  private static final int TEST_TYPE_READ = 0;
+  private static final int TEST_TYPE_WRITE = 1;
+  private static final int TEST_TYPE_CLEANUP = 2;
+  private static final int DEFAULT_BUFFER_SIZE = 1000000;
+  private static final String BASE_FILE_NAME = "test_io_";
+  private static final String DEFAULT_RES_FILE_NAME = "DFSCIOTest_results.log";
+  
+  private static Configuration fsConfig = new Configuration();
+  private static final long MEGA = 0x100000;
+  private static String TEST_ROOT_DIR = System.getProperty("test.build.data","/benchmarks/DFSCIOTest");
+  private static Path CONTROL_DIR = new Path(TEST_ROOT_DIR, "io_control");
+  private static Path WRITE_DIR = new Path(TEST_ROOT_DIR, "io_write");
+  private static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read");
+  private static Path DATA_DIR = new Path(TEST_ROOT_DIR, "io_data");
+
+  private static Path HDFS_TEST_DIR = new Path("/tmp/DFSCIOTest");
+  private static String HDFS_LIB_VERSION = System.getProperty("libhdfs.version", "1");
+  private static String CHMOD = new String("chmod");
+  private static Path HDFS_SHLIB = new Path(HDFS_TEST_DIR + "/libhdfs.so." + HDFS_LIB_VERSION);
+  private static Path HDFS_READ = new Path(HDFS_TEST_DIR + "/hdfs_read");
+  private static Path HDFS_WRITE = new Path(HDFS_TEST_DIR + "/hdfs_write");
+
+  /**
+   * Run the test with default parameters.
+   * 
+   * @throws Exception
+   */
+  public void testIOs() throws Exception {
+    testIOs(10, 10);
+  }
+
+  /**
+   * Run the test with the specified parameters.
+   * 
+   * @param fileSize file size
+   * @param nrFiles number of files
+   * @throws IOException
+   */
+  public static void testIOs(int fileSize, int nrFiles)
+    throws IOException {
+
+    FileSystem fs = FileSystem.get(fsConfig);
+
+    createControlFile(fs, fileSize, nrFiles);
+    writeTest(fs);
+    readTest(fs);
+  }
+
+  private static void createControlFile(
+                                        FileSystem fs,
+                                        int fileSize, // in MB 
+                                        int nrFiles
+                                        ) throws IOException {
+    LOG.info("creating control file: "+fileSize+" mega bytes, "+nrFiles+" files");
+
+    fs.delete(CONTROL_DIR, true);
+
+    for(int i=0; i < nrFiles; i++) {
+      String name = getFileName(i);
+      Path controlFile = new Path(CONTROL_DIR, "in_file_" + name);
+      SequenceFile.Writer writer = null;
+      try {
+        writer = SequenceFile.createWriter(fs, fsConfig, controlFile,
+                                           Text.class, LongWritable.class,
+                                           CompressionType.NONE);
+        writer.append(new Text(name), new LongWritable(fileSize));
+      } catch(Exception e) {
+        throw new IOException(e.getLocalizedMessage());
+      } finally {
+    	if (writer != null)
+          writer.close();
+    	writer = null;
+      }
+    }
+    LOG.info("created control files for: "+nrFiles+" files");
+  }
+
+  private static String getFileName(int fIdx) {
+    return BASE_FILE_NAME + Integer.toString(fIdx);
+  }
+  
+  /**
+   * Write/Read mapper base class.
+   * <p>
+   * Collects the following statistics per task:
+   * <ul>
+   * <li>number of tasks completed</li>
+   * <li>number of bytes written/read</li>
+   * <li>execution time</li>
+   * <li>i/o rate</li>
+   * <li>i/o rate squared</li>
+   * </ul>
+   */
+  private abstract static class IOStatMapper extends IOMapperBase {
+    IOStatMapper() { 
+      super(fsConfig);
+    }
+    
+    void collectStats(OutputCollector<Text, Text> output, 
+                      String name,
+                      long execTime, 
+                      Object objSize) throws IOException {
+      long totalSize = ((Long)objSize).longValue();
+      float ioRateMbSec = (float)totalSize * 1000 / (execTime * MEGA);
+      LOG.info("Number of bytes processed = " + totalSize);
+      LOG.info("Exec time = " + execTime);
+      LOG.info("IO rate = " + ioRateMbSec);
+      
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "tasks"),
+          new Text(String.valueOf(1)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "size"),
+          new Text(String.valueOf(totalSize)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "time"),
+          new Text(String.valueOf(execTime)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "rate"),
+          new Text(String.valueOf(ioRateMbSec*1000)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "sqrate"),
+          new Text(String.valueOf(ioRateMbSec*ioRateMbSec*1000)));
+    }
+  }
+
+  /**
+   * Write mapper class.
+   */
+  public static class WriteMapper extends IOStatMapper {
+
+    public WriteMapper() { 
+      super(); 
+      for(int i=0; i < bufferSize; i++)
+        buffer[i] = (byte)('0' + i % 50);
+    }
+
+    public Object doIO(Reporter reporter, 
+                       String name, 
+                       long totalSize 
+                       ) throws IOException {
+      // create file
+      totalSize *= MEGA;
+      
+      // create instance of local filesystem 
+      FileSystem localFS = FileSystem.getLocal(fsConfig);
+      
+      try {
+        // native runtime
+        Runtime runTime = Runtime.getRuntime();
+          
+        // copy the dso and executable from dfs and chmod them
+        synchronized (this) {
+          localFS.delete(HDFS_TEST_DIR, true);
+          if (!(localFS.mkdirs(HDFS_TEST_DIR))) {
+            throw new IOException("Failed to create " +	HDFS_TEST_DIR + " on local filesystem");
+          }
+        }
+        
+        synchronized (this) {
+          if (!localFS.exists(HDFS_SHLIB)) {
+            FileUtil.copy(fs, HDFS_SHLIB, localFS, HDFS_SHLIB, false, fsConfig);
+
+            String chmodCmd = new String(CHMOD + " a+x " + HDFS_SHLIB);
+            Process process = runTime.exec(chmodCmd);
+            int exitStatus = process.waitFor();
+            if (exitStatus != 0) {
+              throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus);
+            }
+          }
+        } 
+        
+        synchronized (this) {
+          if (!localFS.exists(HDFS_WRITE)) {
+            FileUtil.copy(fs, HDFS_WRITE, localFS, HDFS_WRITE, false, fsConfig);
+
+            String chmodCmd = new String(CHMOD + " a+x " + HDFS_WRITE); 
+            Process process = runTime.exec(chmodCmd);
+            int exitStatus = process.waitFor();
+            if (exitStatus != 0) {
+              throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus);
+            }
+          }
+        }
+    	  	  
+        // exec the C program
+        Path outFile = new Path(DATA_DIR, name);
+        String writeCmd = new String(HDFS_WRITE + " " + outFile + " " + totalSize + " " + bufferSize); 
+        Process process = runTime.exec(writeCmd, null, new File(HDFS_TEST_DIR.toString()));
+        int exitStatus = process.waitFor();
+        if (exitStatus != 0) {
+          throw new IOException(writeCmd + ": Failed with exitStatus: " + exitStatus);
+        }
+      } catch (InterruptedException interruptedException) {
+        reporter.setStatus(interruptedException.toString());
+      } finally {
+        localFS.close();
+      }
+      return new Long(totalSize);
+    }
+  }
+
+  private static void writeTest(FileSystem fs)
+    throws IOException {
+
+    fs.delete(DATA_DIR, true);
+    fs.delete(WRITE_DIR, true);
+    
+    runIOTest(WriteMapper.class, WRITE_DIR);
+  }
+  
+  private static void runIOTest( Class<? extends Mapper> mapperClass, 
+                                 Path outputDir
+                                 ) throws IOException {
+    JobConf job = new JobConf(fsConfig, DFSCIOTest.class);
+
+    FileInputFormat.setInputPaths(job, CONTROL_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(mapperClass);
+    job.setReducerClass(AccumulatingReducer.class);
+
+    FileOutputFormat.setOutputPath(job, outputDir);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+  /**
+   * Read mapper class.
+   */
+  public static class ReadMapper extends IOStatMapper {
+
+    public ReadMapper() { 
+      super(); 
+    }
+
+    public Object doIO(Reporter reporter, 
+                       String name, 
+                       long totalSize 
+                       ) throws IOException {
+      totalSize *= MEGA;
+      
+      // create instance of local filesystem 
+      FileSystem localFS = FileSystem.getLocal(fsConfig);
+      
+      try {
+        // native runtime
+        Runtime runTime = Runtime.getRuntime();
+        
+        // copy the dso and executable from dfs
+        synchronized (this) {
+          localFS.delete(HDFS_TEST_DIR, true);
+          if (!(localFS.mkdirs(HDFS_TEST_DIR))) {
+            throw new IOException("Failed to create " +	HDFS_TEST_DIR + " on local filesystem");
+          }
+        }
+        
+        synchronized (this) {
+          if (!localFS.exists(HDFS_SHLIB)) {
+            if (!FileUtil.copy(fs, HDFS_SHLIB, localFS, HDFS_SHLIB, false, fsConfig)) {
+              throw new IOException("Failed to copy " + HDFS_SHLIB + " to local filesystem");
+            }
+
+            String chmodCmd = new String(CHMOD + " a+x " + HDFS_SHLIB);
+            Process process = runTime.exec(chmodCmd);
+            int exitStatus = process.waitFor();
+            if (exitStatus != 0) {
+              throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus);
+            }
+          }
+        }
+        
+        synchronized (this) {
+          if (!localFS.exists(HDFS_READ)) {
+            if (!FileUtil.copy(fs, HDFS_READ, localFS, HDFS_READ, false, fsConfig)) {
+              throw new IOException("Failed to copy " + HDFS_READ + " to local filesystem");
+            }
+
+            String chmodCmd = new String(CHMOD + " a+x " + HDFS_READ); 
+            Process process = runTime.exec(chmodCmd);
+            int exitStatus = process.waitFor();
+             
+            if (exitStatus != 0) {
+              throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus);
+            }
+          }
+        }
+    	  	  
+        // exec the C program
+        Path inFile = new Path(DATA_DIR, name);
+        String readCmd = new String(HDFS_READ + " " + inFile + " " + totalSize + " " + 
+                                    bufferSize); 
+        Process process = runTime.exec(readCmd, null, new File(HDFS_TEST_DIR.toString()));
+        int exitStatus = process.waitFor();
+        
+        if (exitStatus != 0) {
+          throw new IOException(HDFS_READ + ": Failed with exitStatus: " + exitStatus);
+        }
+      } catch (InterruptedException interruptedException) {
+        reporter.setStatus(interruptedException.toString());
+      } finally {
+        localFS.close();
+      }
+      return new Long(totalSize);
+    }
+  }
+
+  private static void readTest(FileSystem fs) throws IOException {
+    fs.delete(READ_DIR, true);
+    runIOTest(ReadMapper.class, READ_DIR);
+  }
+
+  private static void sequentialTest(
+                                     FileSystem fs, 
+                                     int testType, 
+                                     int fileSize, 
+                                     int nrFiles
+                                     ) throws Exception {
+    IOStatMapper ioer = null;
+    if (testType == TEST_TYPE_READ)
+      ioer = new ReadMapper();
+    else if (testType == TEST_TYPE_WRITE)
+      ioer = new WriteMapper();
+    else
+      return;
+    for(int i=0; i < nrFiles; i++)
+      ioer.doIO(Reporter.NULL,
+                BASE_FILE_NAME+Integer.toString(i), 
+                MEGA*fileSize);
+  }
+
+  public static void main(String[] args) {
+    int testType = TEST_TYPE_READ;
+    int bufferSize = DEFAULT_BUFFER_SIZE;
+    int fileSize = 1;
+    int nrFiles = 1;
+    String resFileName = DEFAULT_RES_FILE_NAME;
+    boolean isSequential = false;
+
+    String version="DFSCIOTest.0.0.1";
+    String usage = "Usage: DFSCIOTest -read | -write | -clean [-nrFiles N] [-fileSize MB] [-resFile resultFileName] [-bufferSize Bytes] ";
+    
+    System.out.println(version);
+    if (args.length == 0) {
+      System.err.println(usage);
+      System.exit(-1);
+    }
+    for (int i = 0; i < args.length; i++) {       // parse command line
+      if (args[i].startsWith("-r")) {
+        testType = TEST_TYPE_READ;
+      } else if (args[i].startsWith("-w")) {
+        testType = TEST_TYPE_WRITE;
+      } else if (args[i].startsWith("-clean")) {
+        testType = TEST_TYPE_CLEANUP;
+      } else if (args[i].startsWith("-seq")) {
+        isSequential = true;
+      } else if (args[i].equals("-nrFiles")) {
+        nrFiles = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-fileSize")) {
+        fileSize = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-bufferSize")) {
+        bufferSize = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-resFile")) {
+        resFileName = args[++i];
+      }
+    }
+
+    LOG.info("nrFiles = " + nrFiles);
+    LOG.info("fileSize (MB) = " + fileSize);
+    LOG.info("bufferSize = " + bufferSize);
+  
+    try {
+      fsConfig.setInt("test.io.file.buffer.size", bufferSize);
+      FileSystem fs = FileSystem.get(fsConfig);
+      
+      if (testType != TEST_TYPE_CLEANUP) {
+        fs.delete(HDFS_TEST_DIR, true);
+        if (!fs.mkdirs(HDFS_TEST_DIR)) {
+          throw new IOException("Mkdirs failed to create " + 
+                                HDFS_TEST_DIR.toString());
+        }
+
+        //Copy the executables over to the remote filesystem
+        String hadoopHome = System.getenv("HADOOP_HOME");
+        fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/libhdfs.so." + HDFS_LIB_VERSION),
+                             HDFS_SHLIB);
+        fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_read"), HDFS_READ);
+        fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_write"), HDFS_WRITE);
+      }
+
+      if (isSequential) {
+        long tStart = System.currentTimeMillis();
+        sequentialTest(fs, testType, fileSize, nrFiles);
+        long execTime = System.currentTimeMillis() - tStart;
+        String resultLine = "Seq Test exec time sec: " + (float)execTime / 1000;
+        LOG.info(resultLine);
+        return;
+      }
+      if (testType == TEST_TYPE_CLEANUP) {
+        cleanup(fs);
+        return;
+      }
+      createControlFile(fs, fileSize, nrFiles);
+      long tStart = System.currentTimeMillis();
+      if (testType == TEST_TYPE_WRITE)
+        writeTest(fs);
+      if (testType == TEST_TYPE_READ)
+        readTest(fs);
+      long execTime = System.currentTimeMillis() - tStart;
+    
+      analyzeResult(fs, testType, execTime, resFileName);
+    } catch(Exception e) {
+      System.err.print(e.getLocalizedMessage());
+      System.exit(-1);
+    }
+  }
+  
+  private static void analyzeResult( FileSystem fs, 
+                                     int testType,
+                                     long execTime,
+                                     String resFileName
+                                     ) throws IOException {
+    Path reduceFile;
+    if (testType == TEST_TYPE_WRITE)
+      reduceFile = new Path(WRITE_DIR, "part-00000");
+    else
+      reduceFile = new Path(READ_DIR, "part-00000");
+    DataInputStream in;
+    in = new DataInputStream(fs.open(reduceFile));
+  
+    BufferedReader lines;
+    lines = new BufferedReader(new InputStreamReader(in));
+    long tasks = 0;
+    long size = 0;
+    long time = 0;
+    float rate = 0;
+    float sqrate = 0;
+    String line;
+    while((line = lines.readLine()) != null) {
+      StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%");
+      String attr = tokens.nextToken(); 
+      if (attr.endsWith(":tasks"))
+        tasks = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith(":size"))
+        size = Long.parseLong(tokens.	nextToken());
+      else if (attr.endsWith(":time"))
+        time = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith(":rate"))
+        rate = Float.parseFloat(tokens.nextToken());
+      else if (attr.endsWith(":sqrate"))
+        sqrate = Float.parseFloat(tokens.nextToken());
+    }
+    
+    double med = rate / 1000 / tasks;
+    double stdDev = Math.sqrt(Math.abs(sqrate / 1000 / tasks - med*med));
+    String resultLines[] = {
+      "----- DFSCIOTest ----- : " + ((testType == TEST_TYPE_WRITE) ? "write" :
+                                     (testType == TEST_TYPE_READ) ? "read" : 
+                                     "unknown"),
+      "           Date & time: " + new Date(System.currentTimeMillis()),
+      "       Number of files: " + tasks,
+      "Total MBytes processed: " + size/MEGA,
+      "     Throughput mb/sec: " + size * 1000.0 / (time * MEGA),
+      "Average IO rate mb/sec: " + med,
+      " Std IO rate deviation: " + stdDev,
+      "    Test exec time sec: " + (float)execTime / 1000,
+      "" };
+
+    PrintStream res = new PrintStream(
+                                      new FileOutputStream(
+                                                           new File(resFileName), true)); 
+    for(int i = 0; i < resultLines.length; i++) {
+      LOG.info(resultLines[i]);
+      res.println(resultLines[i]);
+    }
+  }
+
+  private static void cleanup(FileSystem fs) throws Exception {
+    LOG.info("Cleaning up test files");
+    fs.delete(new Path(TEST_ROOT_DIR), true);
+    fs.delete(HDFS_TEST_DIR, true);
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/DistributedFSCheck.java

@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.Date;
+import java.util.StringTokenizer;
+import java.util.TreeSet;
+import java.util.Vector;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.mapred.*;
+
+/**
+ * Distributed checkup of the file system consistency.
+ * <p>
+ * Test file system consistency by reading each block of each file
+ * of the specified file tree. 
+ * Report corrupted blocks and general file statistics.
+ * <p>
+ * Optionally displays statistics on read performance.
+ * 
+ */
+public class DistributedFSCheck extends TestCase {
+  // Constants
+  private static final Log LOG = LogFactory.getLog(DistributedFSCheck.class);
+  private static final int TEST_TYPE_READ = 0;
+  private static final int TEST_TYPE_CLEANUP = 2;
+  private static final int DEFAULT_BUFFER_SIZE = 1000000;
+  private static final String DEFAULT_RES_FILE_NAME = "DistributedFSCheck_results.log";
+  private static final long MEGA = 0x100000;
+  
+  private static Configuration fsConfig = new Configuration();
+  private static Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","/benchmarks/DistributedFSCheck"));
+  private static Path MAP_INPUT_DIR = new Path(TEST_ROOT_DIR, "map_input");
+  private static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read");
+
+  private FileSystem fs;
+  private long nrFiles;
+  
+  DistributedFSCheck(Configuration conf) throws Exception {
+    fsConfig = conf;
+    this.fs = FileSystem.get(conf);
+  }
+
+  /**
+   * Run distributed checkup for the entire files system.
+   * 
+   * @throws Exception
+   */
+  public void testFSBlocks() throws Exception {
+    testFSBlocks("/");
+  }
+
+  /**
+   * Run distributed checkup for the specified directory.
+   * 
+   * @param rootName root directory name
+   * @throws Exception
+   */
+  public void testFSBlocks(String rootName) throws Exception {
+    createInputFile(rootName);
+    runDistributedFSCheck();
+    cleanup();  // clean up after all to restore the system state
+  }
+
+  private void createInputFile(String rootName) throws IOException {
+    cleanup();  // clean up if previous run failed
+
+    Path inputFile = new Path(MAP_INPUT_DIR, "in_file");
+    SequenceFile.Writer writer =
+      SequenceFile.createWriter(fs, fsConfig, inputFile, 
+                                Text.class, LongWritable.class, CompressionType.NONE);
+    
+    try {
+      nrFiles = 0;
+      listSubtree(new Path(rootName), writer);
+    } finally {
+      writer.close();
+    }
+    LOG.info("Created map input files.");
+  }
+  
+  private void listSubtree(Path rootFile,
+                           SequenceFile.Writer writer
+                           ) throws IOException {
+    FileStatus rootStatus = fs.getFileStatus(rootFile);
+    listSubtree(rootStatus, writer);
+  }
+
+  private void listSubtree(FileStatus rootStatus,
+                           SequenceFile.Writer writer
+                           ) throws IOException {
+    Path rootFile = rootStatus.getPath();
+    if (!rootStatus.isDir()) {
+      nrFiles++;
+      // For a regular file generate <fName,offset> pairs
+      long blockSize = fs.getDefaultBlockSize();
+      long fileLength = rootStatus.getLen();
+      for(long offset = 0; offset < fileLength; offset += blockSize)
+        writer.append(new Text(rootFile.toString()), new LongWritable(offset));
+      return;
+    }
+    
+    FileStatus children[] = fs.listStatus(rootFile);
+    if (children == null)
+      throw new IOException("Could not get listing for " + rootFile);
+    for (int i = 0; i < children.length; i++)
+      listSubtree(children[i], writer);
+  }
+
+  /**
+   * DistributedFSCheck mapper class.
+   */
+  public static class DistributedFSCheckMapper extends IOMapperBase {
+
+    public DistributedFSCheckMapper() { 
+      super(fsConfig); 
+    }
+
+    public Object doIO(Reporter reporter, 
+                       String name, 
+                       long offset 
+                       ) throws IOException {
+      // open file
+      FSDataInputStream in = null;
+      try {
+        in = fs.open(new Path(name));
+      } catch(IOException e) {
+        return name + "@(missing)";
+      }
+      in.seek(offset);
+      long actualSize = 0;
+      try {
+        long blockSize = fs.getDefaultBlockSize();
+        reporter.setStatus("reading " + name + "@" + 
+                           offset + "/" + blockSize);
+        for( int curSize = bufferSize; 
+             curSize == bufferSize && actualSize < blockSize;
+             actualSize += curSize) {
+          curSize = in.read(buffer, 0, bufferSize);
+        }
+      } catch(IOException e) {
+        LOG.info("Corrupted block detected in \"" + name + "\" at " + offset);
+        return name + "@" + offset;
+      } finally {
+        in.close();
+      }
+      return new Long(actualSize);
+    }
+    
+    void collectStats(OutputCollector<Text, Text> output, 
+                      String name, 
+                      long execTime, 
+                      Object corruptedBlock) throws IOException {
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "blocks"),
+          new Text(String.valueOf(1)));
+
+      if (corruptedBlock.getClass().getName().endsWith("String")) {
+        output.collect(
+            new Text(AccumulatingReducer.VALUE_TYPE_STRING + "badBlocks"),
+            new Text((String)corruptedBlock));
+        return;
+      }
+      long totalSize = ((Long)corruptedBlock).longValue();
+      float ioRateMbSec = (float)totalSize * 1000 / (execTime * 0x100000);
+      LOG.info("Number of bytes processed = " + totalSize);
+      LOG.info("Exec time = " + execTime);
+      LOG.info("IO rate = " + ioRateMbSec);
+      
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "size"),
+          new Text(String.valueOf(totalSize)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "time"),
+          new Text(String.valueOf(execTime)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "rate"),
+          new Text(String.valueOf(ioRateMbSec*1000)));
+    }
+  }
+  
+  private void runDistributedFSCheck() throws Exception {
+    JobConf job = new JobConf(fs.getConf(), DistributedFSCheck.class);
+
+    FileInputFormat.setInputPaths(job, MAP_INPUT_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(DistributedFSCheckMapper.class);
+    job.setReducerClass(AccumulatingReducer.class);
+
+    FileOutputFormat.setOutputPath(job, READ_DIR);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+  public static void main(String[] args) throws Exception {
+    int testType = TEST_TYPE_READ;
+    int bufferSize = DEFAULT_BUFFER_SIZE;
+    String resFileName = DEFAULT_RES_FILE_NAME;
+    String rootName = "/";
+    boolean viewStats = false;
+
+    String usage = "Usage: DistributedFSCheck [-root name] [-clean] [-resFile resultFileName] [-bufferSize Bytes] [-stats] ";
+    
+    if (args.length == 1 && args[0].startsWith("-h")) {
+      System.err.println(usage);
+      System.exit(-1);
+    }
+    for(int i = 0; i < args.length; i++) {       // parse command line
+      if (args[i].equals("-root")) {
+        rootName = args[++i];
+      } else if (args[i].startsWith("-clean")) {
+        testType = TEST_TYPE_CLEANUP;
+      } else if (args[i].equals("-bufferSize")) {
+        bufferSize = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-resFile")) {
+        resFileName = args[++i];
+      } else if (args[i].startsWith("-stat")) {
+        viewStats = true;
+      }
+    }
+
+    LOG.info("root = " + rootName);
+    LOG.info("bufferSize = " + bufferSize);
+  
+    Configuration conf = new Configuration();  
+    conf.setInt("test.io.file.buffer.size", bufferSize);
+    DistributedFSCheck test = new DistributedFSCheck(conf);
+
+    if (testType == TEST_TYPE_CLEANUP) {
+      test.cleanup();
+      return;
+    }
+    test.createInputFile(rootName);
+    long tStart = System.currentTimeMillis();
+    test.runDistributedFSCheck();
+    long execTime = System.currentTimeMillis() - tStart;
+    
+    test.analyzeResult(execTime, resFileName, viewStats);
+    // test.cleanup();  // clean up after all to restore the system state
+  }
+  
+  private void analyzeResult(long execTime,
+                             String resFileName,
+                             boolean viewStats
+                             ) throws IOException {
+    Path reduceFile= new Path(READ_DIR, "part-00000");
+    DataInputStream in;
+    in = new DataInputStream(fs.open(reduceFile));
+  
+    BufferedReader lines;
+    lines = new BufferedReader(new InputStreamReader(in));
+    long blocks = 0;
+    long size = 0;
+    long time = 0;
+    float rate = 0;
+    StringTokenizer  badBlocks = null;
+    long nrBadBlocks = 0;
+    String line;
+    while((line = lines.readLine()) != null) {
+      StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%");
+      String attr = tokens.nextToken(); 
+      if (attr.endsWith("blocks"))
+        blocks = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith("size"))
+        size = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith("time"))
+        time = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith("rate"))
+        rate = Float.parseFloat(tokens.nextToken());
+      else if (attr.endsWith("badBlocks")) {
+        badBlocks = new StringTokenizer(tokens.nextToken(), ";");
+        nrBadBlocks = badBlocks.countTokens();
+      }
+    }
+    
+    Vector<String> resultLines = new Vector<String>();
+    resultLines.add( "----- DistributedFSCheck ----- : ");
+    resultLines.add( "               Date & time: " + new Date(System.currentTimeMillis()));
+    resultLines.add( "    Total number of blocks: " + blocks);
+    resultLines.add( "    Total number of  files: " + nrFiles);
+    resultLines.add( "Number of corrupted blocks: " + nrBadBlocks);
+    
+    int nrBadFilesPos = resultLines.size();
+    TreeSet<String> badFiles = new TreeSet<String>();
+    long nrBadFiles = 0;
+    if (nrBadBlocks > 0) {
+      resultLines.add("");
+      resultLines.add("----- Corrupted Blocks (file@offset) ----- : ");
+      while(badBlocks.hasMoreTokens()) {
+        String curBlock = badBlocks.nextToken();
+        resultLines.add(curBlock);
+        badFiles.add(curBlock.substring(0, curBlock.indexOf('@')));
+      }
+      nrBadFiles = badFiles.size();
+    }
+    
+    resultLines.insertElementAt(" Number of corrupted files: " + nrBadFiles, nrBadFilesPos);
+    
+    if (viewStats) {
+      resultLines.add("");
+      resultLines.add("-----   Performance  ----- : ");
+      resultLines.add("         Total MBytes read: " + size/MEGA);
+      resultLines.add("         Throughput mb/sec: " + (float)size * 1000.0 / (time * MEGA));
+      resultLines.add("    Average IO rate mb/sec: " + rate / 1000 / blocks);
+      resultLines.add("        Test exec time sec: " + (float)execTime / 1000);
+    }
+
+    PrintStream res = new PrintStream(
+                                      new FileOutputStream(
+                                                           new File(resFileName), true)); 
+    for(int i = 0; i < resultLines.size(); i++) {
+      String cur = resultLines.get(i);
+      LOG.info(cur);
+      res.println(cur);
+    }
+  }
+
+  private void cleanup() throws IOException {
+    LOG.info("Cleaning up test files");
+    fs.delete(TEST_ROOT_DIR, true);
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/IOMapperBase.java

@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+/**
+ * Base mapper class for IO operations.
+ * <p>
+ * Two abstract method {@link #doIO(Reporter, String, long)} and 
+ * {@link #collectStats(OutputCollector,String,long,Object)} should be
+ * overloaded in derived classes to define the IO operation and the
+ * statistics data to be collected by subsequent reducers.
+ * 
+ */
+public abstract class IOMapperBase extends Configured
+    implements Mapper<Text, LongWritable, Text, Text> {
+  
+  protected byte[] buffer;
+  protected int bufferSize;
+  protected FileSystem fs;
+  protected String hostName;
+
+  public IOMapperBase(Configuration conf) { 
+    super(conf); 
+    try {
+      fs = FileSystem.get(conf);
+    } catch (Exception e) {
+      throw new RuntimeException("Cannot create file system.", e);
+    }
+    bufferSize = conf.getInt("test.io.file.buffer.size", 4096);
+    buffer = new byte[bufferSize];
+    try {
+      hostName = InetAddress.getLocalHost().getHostName();
+    } catch(Exception e) {
+      hostName = "localhost";
+    }
+  }
+
+  public void configure(JobConf job) {
+    setConf(job);
+  }
+
+  public void close() throws IOException {
+  }
+  
+  /**
+   * Perform io operation, usually read or write.
+   * 
+   * @param reporter
+   * @param name file name
+   * @param value offset within the file
+   * @return object that is passed as a parameter to 
+   *          {@link #collectStats(OutputCollector,String,long,Object)}
+   * @throws IOException
+   */
+  abstract Object doIO(Reporter reporter, 
+                       String name, 
+                       long value) throws IOException;
+
+  /**
+   * Collect stat data to be combined by a subsequent reducer.
+   * 
+   * @param output
+   * @param name file name
+   * @param execTime IO execution time
+   * @param doIOReturnValue value returned by {@link #doIO(Reporter,String,long)}
+   * @throws IOException
+   */
+  abstract void collectStats(OutputCollector<Text, Text> output, 
+                             String name, 
+                             long execTime, 
+                             Object doIOReturnValue) throws IOException;
+  
+  /**
+   * Map file name and offset into statistical data.
+   * <p>
+   * The map task is to get the 
+   * <tt>key</tt>, which contains the file name, and the 
+   * <tt>value</tt>, which is the offset within the file.
+   * 
+   * The parameters are passed to the abstract method 
+   * {@link #doIO(Reporter,String,long)}, which performs the io operation, 
+   * usually read or write data, and then 
+   * {@link #collectStats(OutputCollector,String,long,Object)} 
+   * is called to prepare stat data for a subsequent reducer.
+   */
+  public void map(Text key, 
+                  LongWritable value,
+                  OutputCollector<Text, Text> output, 
+                  Reporter reporter) throws IOException {
+    String name = key.toString();
+    long longValue = value.get();
+    
+    reporter.setStatus("starting " + name + " ::host = " + hostName);
+    
+    long tStart = System.currentTimeMillis();
+    Object statValue = doIO(reporter, name, longValue);
+    long tEnd = System.currentTimeMillis();
+    long execTime = tEnd - tStart;
+    collectStats(output, name, execTime, statValue);
+    
+    reporter.setStatus("finished " + name + " ::host = " + hostName);
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java

@@ -0,0 +1,853 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.StringTokenizer;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+
+
+/**
+ * A JUnit test for copying files recursively.
+ */
+public class TestCopyFiles extends TestCase {
+  {
+    ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
+        ).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)DistCp.LOG).getLogger().setLevel(Level.ALL);
+  }
+  
+  static final URI LOCAL_FS = URI.create("file:///");
+  
+  private static final Random RAN = new Random();
+  private static final int NFILES = 20;
+  private static String TEST_ROOT_DIR =
+    new Path(System.getProperty("test.build.data","/tmp"))
+    .toString().replace(' ', '+');
+
+  /** class MyFile contains enough information to recreate the contents of
+   * a single file.
+   */
+  private static class MyFile {
+    private static Random gen = new Random();
+    private static final int MAX_LEVELS = 3;
+    private static final int MAX_SIZE = 8*1024;
+    private static String[] dirNames = {
+      "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"
+    };
+    private final String name;
+    private int size = 0;
+    private long seed = 0L;
+
+    MyFile() {
+      this(gen.nextInt(MAX_LEVELS));
+    }
+    MyFile(int nLevels) {
+      String xname = "";
+      if (nLevels != 0) {
+        int[] levels = new int[nLevels];
+        for (int idx = 0; idx < nLevels; idx++) {
+          levels[idx] = gen.nextInt(10);
+        }
+        StringBuffer sb = new StringBuffer();
+        for (int idx = 0; idx < nLevels; idx++) {
+          sb.append(dirNames[levels[idx]]);
+          sb.append("/");
+        }
+        xname = sb.toString();
+      }
+      long fidx = gen.nextLong() & Long.MAX_VALUE;
+      name = xname + Long.toString(fidx);
+      reset();
+    }
+    void reset() {
+      final int oldsize = size;
+      do { size = gen.nextInt(MAX_SIZE); } while (oldsize == size);
+      final long oldseed = seed;
+      do { seed = gen.nextLong() & Long.MAX_VALUE; } while (oldseed == seed);
+    }
+    String getName() { return name; }
+    int getSize() { return size; }
+    long getSeed() { return seed; }
+  }
+
+  private static MyFile[] createFiles(URI fsname, String topdir)
+    throws IOException {
+    return createFiles(FileSystem.get(fsname, new Configuration()), topdir);
+  }
+
+  /** create NFILES with random names and directory hierarchies
+   * with random (but reproducible) data in them.
+   */
+  private static MyFile[] createFiles(FileSystem fs, String topdir)
+    throws IOException {
+    Path root = new Path(topdir);
+    MyFile[] files = new MyFile[NFILES];
+    for (int i = 0; i < NFILES; i++) {
+      files[i] = createFile(root, fs);
+    }
+    return files;
+  }
+
+  static MyFile createFile(Path root, FileSystem fs, int levels)
+      throws IOException {
+    MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
+    Path p = new Path(root, f.getName());
+    FSDataOutputStream out = fs.create(p);
+    byte[] toWrite = new byte[f.getSize()];
+    new Random(f.getSeed()).nextBytes(toWrite);
+    out.write(toWrite);
+    out.close();
+    FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
+    return f;
+  }
+
+  static MyFile createFile(Path root, FileSystem fs) throws IOException {
+    return createFile(root, fs, -1);
+  }
+
+  private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files
+      ) throws IOException {
+    return checkFiles(fs, topdir, files, false);    
+  }
+
+  private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files,
+      boolean existingOnly) throws IOException {
+    Path root = new Path(topdir);
+    
+    for (int idx = 0; idx < files.length; idx++) {
+      Path fPath = new Path(root, files[idx].getName());
+      try {
+        fs.getFileStatus(fPath);
+        FSDataInputStream in = fs.open(fPath);
+        byte[] toRead = new byte[files[idx].getSize()];
+        byte[] toCompare = new byte[files[idx].getSize()];
+        Random rb = new Random(files[idx].getSeed());
+        rb.nextBytes(toCompare);
+        assertEquals("Cannnot read file.", toRead.length, in.read(toRead));
+        in.close();
+        for (int i = 0; i < toRead.length; i++) {
+          if (toRead[i] != toCompare[i]) {
+            return false;
+          }
+        }
+        toRead = null;
+        toCompare = null;
+      }
+      catch(FileNotFoundException fnfe) {
+        if (!existingOnly) {
+          throw fnfe;
+        }
+      }
+    }
+    
+    return true;
+  }
+
+  private static void updateFiles(FileSystem fs, String topdir, MyFile[] files,
+        int nupdate) throws IOException {
+    assert nupdate <= NFILES;
+
+    Path root = new Path(topdir);
+
+    for (int idx = 0; idx < nupdate; ++idx) {
+      Path fPath = new Path(root, files[idx].getName());
+      // overwrite file
+      assertTrue(fPath.toString() + " does not exist", fs.exists(fPath));
+      FSDataOutputStream out = fs.create(fPath);
+      files[idx].reset();
+      byte[] toWrite = new byte[files[idx].getSize()];
+      Random rb = new Random(files[idx].getSeed());
+      rb.nextBytes(toWrite);
+      out.write(toWrite);
+      out.close();
+    }
+  }
+
+  private static FileStatus[] getFileStatus(FileSystem fs,
+      String topdir, MyFile[] files) throws IOException {
+    return getFileStatus(fs, topdir, files, false);
+  }
+  private static FileStatus[] getFileStatus(FileSystem fs,
+      String topdir, MyFile[] files, boolean existingOnly) throws IOException {
+    Path root = new Path(topdir);
+    List<FileStatus> statuses = new ArrayList<FileStatus>();
+    for (int idx = 0; idx < NFILES; ++idx) {
+      try {
+        statuses.add(fs.getFileStatus(new Path(root, files[idx].getName())));
+      } catch(FileNotFoundException fnfe) {
+        if (!existingOnly) {
+          throw fnfe;
+        }
+      }
+    }
+    return statuses.toArray(new FileStatus[statuses.size()]);
+  }
+
+  private static boolean checkUpdate(FileSystem fs, FileStatus[] old,
+      String topdir, MyFile[] upd, final int nupdate) throws IOException {
+    Path root = new Path(topdir);
+
+    // overwrote updated files
+    for (int idx = 0; idx < nupdate; ++idx) {
+      final FileStatus stat =
+        fs.getFileStatus(new Path(root, upd[idx].getName()));
+      if (stat.getModificationTime() <= old[idx].getModificationTime()) {
+        return false;
+      }
+    }
+    // did not overwrite files not updated
+    for (int idx = nupdate; idx < NFILES; ++idx) {
+      final FileStatus stat =
+        fs.getFileStatus(new Path(root, upd[idx].getName()));
+      if (stat.getModificationTime() != old[idx].getModificationTime()) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /** delete directory and everything underneath it.*/
+  private static void deldir(FileSystem fs, String topdir) throws IOException {
+    fs.delete(new Path(topdir), true);
+  }
+  
+  /** copy files from local file system to local file system */
+  public void testCopyFromLocalToLocal() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
+    MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
+    ToolRunner.run(new DistCp(new Configuration()),
+                           new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
+                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
+    assertTrue("Source and destination directories do not match.",
+               checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
+    deldir(localfs, TEST_ROOT_DIR+"/destdat");
+    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
+  }
+  
+  /** copy files from dfs file system to dfs file system */
+  public void testCopyFromDfsToDfs() throws Exception {
+    String namenode = null;
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      final FileSystem hdfs = cluster.getFileSystem();
+      namenode = FileSystem.getDefaultUri(conf).toString();
+      if (namenode.startsWith("hdfs://")) {
+        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-log",
+                                         namenode+"/logs",
+                                         namenode+"/srcdat",
+                                         namenode+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(hdfs, "/destdat", files));
+        FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
+        assertTrue("Log directory does not exist.",
+                   fs.exists(new Path(namenode+"/logs")));
+        deldir(hdfs, "/destdat");
+        deldir(hdfs, "/srcdat");
+        deldir(hdfs, "/logs");
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+  
+  /** copy files from local file system to dfs file system */
+  public void testCopyFromLocalToDfs() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, 1, true, null);
+      final FileSystem hdfs = cluster.getFileSystem();
+      final String namenode = hdfs.getUri().toString();
+      if (namenode.startsWith("hdfs://")) {
+        MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-log",
+                                         namenode+"/logs",
+                                         "file:///"+TEST_ROOT_DIR+"/srcdat",
+                                         namenode+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(cluster.getFileSystem(), "/destdat", files));
+        assertTrue("Log directory does not exist.",
+                    hdfs.exists(new Path(namenode+"/logs")));
+        deldir(hdfs, "/destdat");
+        deldir(hdfs, "/logs");
+        deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  /** copy files from dfs file system to local file system */
+  public void testCopyFromDfsToLocal() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
+      cluster = new MiniDFSCluster(conf, 1, true, null);
+      final FileSystem hdfs = cluster.getFileSystem();
+      final String namenode = FileSystem.getDefaultUri(conf).toString();
+      if (namenode.startsWith("hdfs://")) {
+        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-log",
+                                         "/logs",
+                                         namenode+"/srcdat",
+                                         "file:///"+TEST_ROOT_DIR+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
+        assertTrue("Log directory does not exist.",
+                    hdfs.exists(new Path("/logs")));
+        deldir(localfs, TEST_ROOT_DIR+"/destdat");
+        deldir(hdfs, "/logs");
+        deldir(hdfs, "/srcdat");
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      final FileSystem hdfs = cluster.getFileSystem();
+      final String namenode = hdfs.getUri().toString();
+      if (namenode.startsWith("hdfs://")) {
+        MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-p",
+                                         "-log",
+                                         namenode+"/logs",
+                                         namenode+"/srcdat",
+                                         namenode+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(hdfs, "/destdat", files));
+        FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
+        assertTrue("Log directory does not exist.",
+                    fs.exists(new Path(namenode+"/logs")));
+
+        FileStatus[] dchkpoint = getFileStatus(hdfs, "/destdat", files);
+        final int nupdate = NFILES>>2;
+        updateFiles(cluster.getFileSystem(), "/srcdat", files, nupdate);
+        deldir(hdfs, "/logs");
+
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-p",
+                                         "-update",
+                                         "-log",
+                                         namenode+"/logs",
+                                         namenode+"/srcdat",
+                                         namenode+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(hdfs, "/destdat", files));
+        assertTrue("Update failed to replicate all changes in src",
+                 checkUpdate(hdfs, dchkpoint, "/destdat", files, nupdate));
+
+        deldir(hdfs, "/logs");
+        ToolRunner.run(new DistCp(conf), new String[] {
+                                         "-p",
+                                         "-overwrite",
+                                         "-log",
+                                         namenode+"/logs",
+                                         namenode+"/srcdat",
+                                         namenode+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(hdfs, "/destdat", files));
+        assertTrue("-overwrite didn't.",
+                 checkUpdate(hdfs, dchkpoint, "/destdat", files, NFILES));
+
+        deldir(hdfs, "/destdat");
+        deldir(hdfs, "/srcdat");
+        deldir(hdfs, "/logs");
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  public void testCopyDuplication() throws Exception {
+    final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
+    try {    
+      MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
+      ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
+                        "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
+      assertTrue("Source and destination directories do not match.",
+                 checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));
+  
+      assertEquals(DistCp.DuplicationException.ERROR_CODE,
+          ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
+                        "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
+                        "file:///"+TEST_ROOT_DIR+"/destdat",}));
+    }
+    finally {
+      deldir(localfs, TEST_ROOT_DIR+"/destdat");
+      deldir(localfs, TEST_ROOT_DIR+"/srcdat");
+      deldir(localfs, TEST_ROOT_DIR+"/src2");
+    }
+  }
+
+  public void testCopySingleFile() throws Exception {
+    FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
+    Path root = new Path(TEST_ROOT_DIR+"/srcdat");
+    try {    
+      MyFile[] files = {createFile(root, fs)};
+      //copy a dir with a single file
+      ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
+                        "file:///"+TEST_ROOT_DIR+"/destdat"});
+      assertTrue("Source and destination directories do not match.",
+                 checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
+      
+      //copy a single file
+      String fname = files[0].getName();
+      Path p = new Path(root, fname);
+      FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
+      ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));     
+      //copy single file to existing dir
+      deldir(fs, TEST_ROOT_DIR+"/dest2");
+      fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
+      MyFile[] files2 = {createFile(root, fs, 0)};
+      String sname = files2[0].getName();
+      ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
+      updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
+      //copy single file to existing dir w/ dst name conflict
+      ToolRunner.run(new DistCp(new Configuration()),
+          new String[] {"-update",
+                        "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+                        "file:///"+TEST_ROOT_DIR+"/dest2/"});
+      assertTrue("Source and destination directories do not match.",
+          checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));     
+    }
+    finally {
+      deldir(fs, TEST_ROOT_DIR+"/destdat");
+      deldir(fs, TEST_ROOT_DIR+"/dest2");
+      deldir(fs, TEST_ROOT_DIR+"/srcdat");
+    }
+  }
+
+  public void testPreserveOption() throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      String nnUri = FileSystem.getDefaultUri(conf).toString();
+      FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
+
+      {//test preserving user
+        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
+        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
+        for(int i = 0; i < srcstat.length; i++) {
+          fs.setOwner(srcstat[i].getPath(), "u" + i, null);
+        }
+        ToolRunner.run(new DistCp(conf),
+            new String[]{"-pu", nnUri+"/srcdat", nnUri+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(fs, "/destdat", files));
+        
+        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
+        for(int i = 0; i < dststat.length; i++) {
+          assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
+        }
+        deldir(fs, "/destdat");
+        deldir(fs, "/srcdat");
+      }
+
+      {//test preserving group
+        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
+        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
+        for(int i = 0; i < srcstat.length; i++) {
+          fs.setOwner(srcstat[i].getPath(), null, "g" + i);
+        }
+        ToolRunner.run(new DistCp(conf),
+            new String[]{"-pg", nnUri+"/srcdat", nnUri+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(fs, "/destdat", files));
+        
+        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
+        for(int i = 0; i < dststat.length; i++) {
+          assertEquals("i=" + i, "g" + i, dststat[i].getGroup());
+        }
+        deldir(fs, "/destdat");
+        deldir(fs, "/srcdat");
+      }
+
+      {//test preserving mode
+        MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
+        FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
+        FsPermission[] permissions = new FsPermission[srcstat.length];
+        for(int i = 0; i < srcstat.length; i++) {
+          permissions[i] = new FsPermission((short)(i & 0666));
+          fs.setPermission(srcstat[i].getPath(), permissions[i]);
+        }
+
+        ToolRunner.run(new DistCp(conf),
+            new String[]{"-pp", nnUri+"/srcdat", nnUri+"/destdat"});
+        assertTrue("Source and destination directories do not match.",
+                   checkFiles(fs, "/destdat", files));
+  
+        FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
+        for(int i = 0; i < dststat.length; i++) {
+          assertEquals("i=" + i, permissions[i], dststat[i].getPermission());
+        }
+        deldir(fs, "/destdat");
+        deldir(fs, "/srcdat");
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  public void testMapCount() throws Exception {
+    String namenode = null;
+    MiniDFSCluster dfs = null;
+    MiniMRCluster mr = null;
+    try {
+      Configuration conf = new Configuration();
+      dfs = new MiniDFSCluster(conf, 3, true, null);
+      FileSystem fs = dfs.getFileSystem();
+      final FsShell shell = new FsShell(conf);
+      namenode = fs.getUri().toString();
+      mr = new MiniMRCluster(3, namenode, 1);
+      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
+      long totsize = 0;
+      for (MyFile f : files) {
+        totsize += f.getSize();
+      }
+      Configuration job = mr.createJobConf();
+      job.setLong("distcp.bytes.per.map", totsize / 3);
+      ToolRunner.run(new DistCp(job),
+          new String[] {"-m", "100",
+                        "-log",
+                        namenode+"/logs",
+                        namenode+"/srcdat",
+                        namenode+"/destdat"});
+      assertTrue("Source and destination directories do not match.",
+                 checkFiles(fs, "/destdat", files));
+
+      String logdir = namenode + "/logs";
+      System.out.println(execCmd(shell, "-lsr", logdir));
+      FileStatus[] logs = fs.listStatus(new Path(logdir));
+      // rare case where splits are exact, logs.length can be 4
+      assertTrue("Unexpected map count, logs.length=" + logs.length,
+          logs.length == 5 || logs.length == 4);
+
+      deldir(fs, "/destdat");
+      deldir(fs, "/logs");
+      ToolRunner.run(new DistCp(job),
+          new String[] {"-m", "1",
+                        "-log",
+                        namenode+"/logs",
+                        namenode+"/srcdat",
+                        namenode+"/destdat"});
+
+      System.out.println(execCmd(shell, "-lsr", logdir));
+      logs = fs.listStatus(new Path(namenode+"/logs"));
+      assertTrue("Unexpected map count, logs.length=" + logs.length,
+          logs.length == 2);
+    } finally {
+      if (dfs != null) { dfs.shutdown(); }
+      if (mr != null) { mr.shutdown(); }
+    }
+  }
+
+  public void testLimits() throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      final String nnUri = FileSystem.getDefaultUri(conf).toString();
+      final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
+      final DistCp distcp = new DistCp(conf);
+      final FsShell shell = new FsShell(conf);  
+
+      final String srcrootdir =  "/src_root";
+      final Path srcrootpath = new Path(srcrootdir); 
+      final String dstrootdir =  "/dst_root";
+      final Path dstrootpath = new Path(dstrootdir); 
+
+      {//test -filelimit
+        MyFile[] files = createFiles(URI.create(nnUri), srcrootdir);
+        int filelimit = files.length / 2;
+        System.out.println("filelimit=" + filelimit);
+
+        ToolRunner.run(distcp,
+            new String[]{"-filelimit", ""+filelimit, nnUri+srcrootdir, nnUri+dstrootdir});
+        String results = execCmd(shell, "-lsr", dstrootdir);
+        results = removePrefix(results, dstrootdir);
+        System.out.println("results=" +  results);
+
+        FileStatus[] dststat = getFileStatus(fs, dstrootdir, files, true);
+        assertEquals(filelimit, dststat.length);
+        deldir(fs, dstrootdir);
+        deldir(fs, srcrootdir);
+      }
+
+      {//test -sizelimit
+        createFiles(URI.create(nnUri), srcrootdir);
+        long sizelimit = fs.getContentSummary(srcrootpath).getLength()/2;
+        System.out.println("sizelimit=" + sizelimit);
+
+        ToolRunner.run(distcp,
+            new String[]{"-sizelimit", ""+sizelimit, nnUri+srcrootdir, nnUri+dstrootdir});
+        
+        ContentSummary summary = fs.getContentSummary(dstrootpath);
+        System.out.println("summary=" + summary);
+        assertTrue(summary.getLength() <= sizelimit);
+        deldir(fs, dstrootdir);
+        deldir(fs, srcrootdir);
+      }
+
+      {//test update
+        final MyFile[] srcs = createFiles(URI.create(nnUri), srcrootdir);
+        final long totalsize = fs.getContentSummary(srcrootpath).getLength();
+        System.out.println("src.length=" + srcs.length);
+        System.out.println("totalsize =" + totalsize);
+        fs.mkdirs(dstrootpath);
+        final int parts = RAN.nextInt(NFILES/3 - 1) + 2;
+        final int filelimit = srcs.length/parts;
+        final long sizelimit = totalsize/parts;
+        System.out.println("filelimit=" + filelimit);
+        System.out.println("sizelimit=" + sizelimit);
+        System.out.println("parts    =" + parts);
+        final String[] args = {"-filelimit", ""+filelimit, "-sizelimit", ""+sizelimit,
+            "-update", nnUri+srcrootdir, nnUri+dstrootdir};
+
+        int dstfilecount = 0;
+        long dstsize = 0;
+        for(int i = 0; i <= parts; i++) {
+          ToolRunner.run(distcp, args);
+        
+          FileStatus[] dststat = getFileStatus(fs, dstrootdir, srcs, true);
+          System.out.println(i + ") dststat.length=" + dststat.length);
+          assertTrue(dststat.length - dstfilecount <= filelimit);
+          ContentSummary summary = fs.getContentSummary(dstrootpath);
+          System.out.println(i + ") summary.getLength()=" + summary.getLength());
+          assertTrue(summary.getLength() - dstsize <= sizelimit);
+          assertTrue(checkFiles(fs, dstrootdir, srcs, true));
+          dstfilecount = dststat.length;
+          dstsize = summary.getLength();
+        }
+
+        deldir(fs, dstrootdir);
+        deldir(fs, srcrootdir);
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  static final long now = System.currentTimeMillis();
+
+  static UnixUserGroupInformation createUGI(String name, boolean issuper) {
+    String username = name + now;
+    String group = issuper? "supergroup": username;
+    return UnixUserGroupInformation.createImmutable(
+        new String[]{username, group});
+  }
+
+  static Path createHomeDirectory(FileSystem fs, UserGroupInformation ugi
+      ) throws IOException {
+    final Path home = new Path("/user/" + ugi.getUserName());
+    fs.mkdirs(home);
+    fs.setOwner(home, ugi.getUserName(), ugi.getGroupNames()[0]);
+    fs.setPermission(home, new FsPermission((short)0700));
+    return home;
+  }
+
+  public void testHftpAccessControl() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      final UnixUserGroupInformation DFS_UGI = createUGI("dfs", true); 
+      final UnixUserGroupInformation USER_UGI = createUGI("user", false); 
+
+      //start cluster by DFS_UGI
+      final Configuration dfsConf = new Configuration();
+      UnixUserGroupInformation.saveToConf(dfsConf,
+          UnixUserGroupInformation.UGI_PROPERTY_NAME, DFS_UGI);
+      cluster = new MiniDFSCluster(dfsConf, 2, true, null);
+      cluster.waitActive();
+
+      final String httpAdd = dfsConf.get("dfs.http.address");
+      final URI nnURI = FileSystem.getDefaultUri(dfsConf);
+      final String nnUri = nnURI.toString();
+      final Path home = createHomeDirectory(FileSystem.get(nnURI, dfsConf), USER_UGI);
+      
+      //now, login as USER_UGI
+      final Configuration userConf = new Configuration();
+      UnixUserGroupInformation.saveToConf(userConf,
+          UnixUserGroupInformation.UGI_PROPERTY_NAME, USER_UGI);
+      final FileSystem fs = FileSystem.get(nnURI, userConf);
+
+      final Path srcrootpath = new Path(home, "src_root"); 
+      final String srcrootdir =  srcrootpath.toString();
+      final Path dstrootpath = new Path(home, "dst_root"); 
+      final String dstrootdir =  dstrootpath.toString();
+      final DistCp distcp = new DistCp(userConf);
+
+      FileSystem.mkdirs(fs, srcrootpath, new FsPermission((short)0700));
+      final String[] args = {"hftp://"+httpAdd+srcrootdir, nnUri+dstrootdir};
+
+      { //copy with permission 000, should fail
+        fs.setPermission(srcrootpath, new FsPermission((short)0));
+        assertEquals(-3, ToolRunner.run(distcp, args));
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  /** test -delete */
+  public void testDelete() throws Exception {
+    final Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      final URI nnURI = FileSystem.getDefaultUri(conf);
+      final String nnUri = nnURI.toString();
+      final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
+
+      final DistCp distcp = new DistCp(conf);
+      final FsShell shell = new FsShell(conf);  
+
+      final String srcrootdir = "/src_root";
+      final String dstrootdir = "/dst_root";
+
+      {
+        //create source files
+        createFiles(nnURI, srcrootdir);
+        String srcresults = execCmd(shell, "-lsr", srcrootdir);
+        srcresults = removePrefix(srcresults, srcrootdir);
+        System.out.println("srcresults=" +  srcresults);
+
+        //create some files in dst
+        createFiles(nnURI, dstrootdir);
+        System.out.println("dstrootdir=" +  dstrootdir);
+        shell.run(new String[]{"-lsr", dstrootdir});
+
+        //run distcp
+        ToolRunner.run(distcp,
+            new String[]{"-delete", "-update", "-log", "/log",
+                         nnUri+srcrootdir, nnUri+dstrootdir});
+
+        //make sure src and dst contains the same files
+        String dstresults = execCmd(shell, "-lsr", dstrootdir);
+        dstresults = removePrefix(dstresults, dstrootdir);
+        System.out.println("first dstresults=" +  dstresults);
+        assertEquals(srcresults, dstresults);
+
+        //create additional file in dst
+        create(fs, new Path(dstrootdir, "foo"));
+        create(fs, new Path(dstrootdir, "foobar"));
+
+        //run distcp again
+        ToolRunner.run(distcp,
+            new String[]{"-delete", "-update", "-log", "/log2",
+                         nnUri+srcrootdir, nnUri+dstrootdir});
+        
+        //make sure src and dst contains the same files
+        dstresults = execCmd(shell, "-lsr", dstrootdir);
+        dstresults = removePrefix(dstresults, dstrootdir);
+        System.out.println("second dstresults=" +  dstresults);
+        assertEquals(srcresults, dstresults);
+
+        //cleanup
+        deldir(fs, dstrootdir);
+        deldir(fs, srcrootdir);
+      }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  static void create(FileSystem fs, Path f) throws IOException {
+    FSDataOutputStream out = fs.create(f);
+    try {
+      byte[] b = new byte[1024 + RAN.nextInt(1024)];
+      RAN.nextBytes(b);
+      out.write(b);
+    } finally {
+      if (out != null) out.close();
+    }
+  }
+  
+  static String execCmd(FsShell shell, String... args) throws Exception {
+    ByteArrayOutputStream baout = new ByteArrayOutputStream();
+    PrintStream out = new PrintStream(baout, true);
+    PrintStream old = System.out;
+    System.setOut(out);
+    shell.run(args);
+    out.close();
+    System.setOut(old);
+    return baout.toString();
+  }
+  
+  private static String removePrefix(String lines, String prefix) {
+    final int prefixlen = prefix.length();
+    final StringTokenizer t = new StringTokenizer(lines, "\n");
+    final StringBuffer results = new StringBuffer(); 
+    for(; t.hasMoreTokens(); ) {
+      String s = t.nextToken();
+      results.append(s.substring(s.indexOf(prefix) + prefixlen) + "\n");
+    }
+    return results.toString();
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/TestDFSIO.java

@@ -0,0 +1,445 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Date;
+import java.util.StringTokenizer;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Distributed i/o benchmark.
+ * <p>
+ * This test writes into or reads from a specified number of files.
+ * File size is specified as a parameter to the test. 
+ * Each file is accessed in a separate map task.
+ * <p>
+ * The reducer collects the following statistics:
+ * <ul>
+ * <li>number of tasks completed</li>
+ * <li>number of bytes written/read</li>
+ * <li>execution time</li>
+ * <li>io rate</li>
+ * <li>io rate squared</li>
+ * </ul>
+ *    
+ * Finally, the following information is appended to a local file
+ * <ul>
+ * <li>read or write test</li>
+ * <li>date and time the test finished</li>   
+ * <li>number of files</li>
+ * <li>total number of bytes processed</li>
+ * <li>throughput in mb/sec (total number of bytes / sum of processing times)</li>
+ * <li>average i/o rate in mb/sec per file</li>
+ * <li>standard deviation of i/o rate </li>
+ * </ul>
+ */
+public class TestDFSIO extends TestCase {
+  // Constants
+  private static final Log LOG = LogFactory.getLog(TestDFSIO.class);
+  private static final int TEST_TYPE_READ = 0;
+  private static final int TEST_TYPE_WRITE = 1;
+  private static final int TEST_TYPE_CLEANUP = 2;
+  private static final int DEFAULT_BUFFER_SIZE = 1000000;
+  private static final String BASE_FILE_NAME = "test_io_";
+  private static final String DEFAULT_RES_FILE_NAME = "TestDFSIO_results.log";
+  
+  private static Configuration fsConfig = new Configuration();
+  private static final long MEGA = 0x100000;
+  private static String TEST_ROOT_DIR = System.getProperty("test.build.data","/benchmarks/TestDFSIO");
+  private static Path CONTROL_DIR = new Path(TEST_ROOT_DIR, "io_control");
+  private static Path WRITE_DIR = new Path(TEST_ROOT_DIR, "io_write");
+  private static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read");
+  private static Path DATA_DIR = new Path(TEST_ROOT_DIR, "io_data");
+
+  /**
+   * Run the test with default parameters.
+   * 
+   * @throws Exception
+   */
+  public void testIOs() throws Exception {
+    testIOs(10, 10);
+  }
+
+  /**
+   * Run the test with the specified parameters.
+   * 
+   * @param fileSize file size
+   * @param nrFiles number of files
+   * @throws IOException
+   */
+  public static void testIOs(int fileSize, int nrFiles)
+    throws IOException {
+
+    FileSystem fs = FileSystem.get(fsConfig);
+
+    createControlFile(fs, fileSize, nrFiles);
+    writeTest(fs);
+    readTest(fs);
+    cleanup(fs);
+  }
+
+  private static void createControlFile(
+                                        FileSystem fs,
+                                        int fileSize, // in MB 
+                                        int nrFiles
+                                        ) throws IOException {
+    LOG.info("creating control file: "+fileSize+" mega bytes, "+nrFiles+" files");
+
+    fs.delete(CONTROL_DIR, true);
+
+    for(int i=0; i < nrFiles; i++) {
+      String name = getFileName(i);
+      Path controlFile = new Path(CONTROL_DIR, "in_file_" + name);
+      SequenceFile.Writer writer = null;
+      try {
+        writer = SequenceFile.createWriter(fs, fsConfig, controlFile,
+                                           Text.class, LongWritable.class,
+                                           CompressionType.NONE);
+        writer.append(new Text(name), new LongWritable(fileSize));
+      } catch(Exception e) {
+        throw new IOException(e.getLocalizedMessage());
+      } finally {
+    	if (writer != null)
+          writer.close();
+    	writer = null;
+      }
+    }
+    LOG.info("created control files for: "+nrFiles+" files");
+  }
+
+  private static String getFileName(int fIdx) {
+    return BASE_FILE_NAME + Integer.toString(fIdx);
+  }
+  
+  /**
+   * Write/Read mapper base class.
+   * <p>
+   * Collects the following statistics per task:
+   * <ul>
+   * <li>number of tasks completed</li>
+   * <li>number of bytes written/read</li>
+   * <li>execution time</li>
+   * <li>i/o rate</li>
+   * <li>i/o rate squared</li>
+   * </ul>
+   */
+  private abstract static class IOStatMapper extends IOMapperBase {
+    IOStatMapper() { 
+      super(fsConfig);
+    }
+    
+    void collectStats(OutputCollector<Text, Text> output, 
+                      String name,
+                      long execTime, 
+                      Object objSize) throws IOException {
+      long totalSize = ((Long)objSize).longValue();
+      float ioRateMbSec = (float)totalSize * 1000 / (execTime * MEGA);
+      LOG.info("Number of bytes processed = " + totalSize);
+      LOG.info("Exec time = " + execTime);
+      LOG.info("IO rate = " + ioRateMbSec);
+      
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "tasks"),
+          new Text(String.valueOf(1)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "size"),
+          new Text(String.valueOf(totalSize)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_LONG + "time"),
+          new Text(String.valueOf(execTime)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "rate"),
+          new Text(String.valueOf(ioRateMbSec*1000)));
+      output.collect(new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "sqrate"),
+          new Text(String.valueOf(ioRateMbSec*ioRateMbSec*1000)));
+    }
+  }
+
+  /**
+   * Write mapper class.
+   */
+  public static class WriteMapper extends IOStatMapper {
+
+    public WriteMapper() { 
+      super(); 
+      for(int i=0; i < bufferSize; i++)
+        buffer[i] = (byte)('0' + i % 50);
+    }
+
+    public Object doIO(Reporter reporter, 
+                       String name, 
+                       long totalSize 
+                       ) throws IOException {
+      // create file
+      totalSize *= MEGA;
+      OutputStream out;
+      out = fs.create(new Path(DATA_DIR, name), true, bufferSize);
+      
+      try {
+        // write to the file
+        long nrRemaining;
+        for (nrRemaining = totalSize; nrRemaining > 0; nrRemaining -= bufferSize) {
+          int curSize = (bufferSize < nrRemaining) ? bufferSize : (int)nrRemaining; 
+          out.write(buffer, 0, curSize);
+          reporter.setStatus("writing " + name + "@" + 
+                             (totalSize - nrRemaining) + "/" + totalSize 
+                             + " ::host = " + hostName);
+        }
+      } finally {
+        out.close();
+      }
+      return new Long(totalSize);
+    }
+  }
+
+  private static void writeTest(FileSystem fs)
+    throws IOException {
+
+    fs.delete(DATA_DIR, true);
+    fs.delete(WRITE_DIR, true);
+    
+    runIOTest(WriteMapper.class, WRITE_DIR);
+  }
+  
+  private static void runIOTest( Class<? extends Mapper> mapperClass, 
+                                 Path outputDir
+                                 ) throws IOException {
+    JobConf job = new JobConf(fsConfig, TestDFSIO.class);
+
+    FileInputFormat.setInputPaths(job, CONTROL_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(mapperClass);
+    job.setReducerClass(AccumulatingReducer.class);
+
+    FileOutputFormat.setOutputPath(job, outputDir);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+  /**
+   * Read mapper class.
+   */
+  public static class ReadMapper extends IOStatMapper {
+
+    public ReadMapper() { 
+      super(); 
+    }
+
+    public Object doIO(Reporter reporter, 
+                       String name, 
+                       long totalSize 
+                       ) throws IOException {
+      totalSize *= MEGA;
+      // open file
+      DataInputStream in = fs.open(new Path(DATA_DIR, name));
+      try {
+        long actualSize = 0;
+        for(int curSize = bufferSize; curSize == bufferSize;) {
+          curSize = in.read(buffer, 0, bufferSize);
+          actualSize += curSize;
+          reporter.setStatus("reading " + name + "@" + 
+                             actualSize + "/" + totalSize 
+                             + " ::host = " + hostName);
+        }
+      } finally {
+        in.close();
+      }
+      return new Long(totalSize);
+    }
+  }
+
+  private static void readTest(FileSystem fs) throws IOException {
+    fs.delete(READ_DIR, true);
+    runIOTest(ReadMapper.class, READ_DIR);
+  }
+
+  private static void sequentialTest(
+                                     FileSystem fs, 
+                                     int testType, 
+                                     int fileSize, 
+                                     int nrFiles
+                                     ) throws Exception {
+    IOStatMapper ioer = null;
+    if (testType == TEST_TYPE_READ)
+      ioer = new ReadMapper();
+    else if (testType == TEST_TYPE_WRITE)
+      ioer = new WriteMapper();
+    else
+      return;
+    for(int i=0; i < nrFiles; i++)
+      ioer.doIO(Reporter.NULL,
+                BASE_FILE_NAME+Integer.toString(i), 
+                MEGA*fileSize);
+  }
+
+  public static void main(String[] args) {
+    int testType = TEST_TYPE_READ;
+    int bufferSize = DEFAULT_BUFFER_SIZE;
+    int fileSize = 1;
+    int nrFiles = 1;
+    String resFileName = DEFAULT_RES_FILE_NAME;
+    boolean isSequential = false;
+    
+    String className = TestDFSIO.class.getSimpleName();
+    String version = className + ".0.0.4";
+    String usage = "Usage: " + className + " -read | -write | -clean [-nrFiles N] [-fileSize MB] [-resFile resultFileName] [-bufferSize Bytes] ";
+    
+    System.out.println(version);
+    if (args.length == 0) {
+      System.err.println(usage);
+      System.exit(-1);
+    }
+    for (int i = 0; i < args.length; i++) {       // parse command line
+      if (args[i].startsWith("-read")) {
+        testType = TEST_TYPE_READ;
+      } else if (args[i].equals("-write")) {
+        testType = TEST_TYPE_WRITE;
+      } else if (args[i].equals("-clean")) {
+        testType = TEST_TYPE_CLEANUP;
+      } else if (args[i].startsWith("-seq")) {
+        isSequential = true;
+      } else if (args[i].equals("-nrFiles")) {
+        nrFiles = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-fileSize")) {
+        fileSize = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-bufferSize")) {
+        bufferSize = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-resFile")) {
+        resFileName = args[++i];
+      }
+    }
+
+    LOG.info("nrFiles = " + nrFiles);
+    LOG.info("fileSize (MB) = " + fileSize);
+    LOG.info("bufferSize = " + bufferSize);
+  
+    try {
+      fsConfig.setInt("test.io.file.buffer.size", bufferSize);
+      FileSystem fs = FileSystem.get(fsConfig);
+
+      if (isSequential) {
+        long tStart = System.currentTimeMillis();
+        sequentialTest(fs, testType, fileSize, nrFiles);
+        long execTime = System.currentTimeMillis() - tStart;
+        String resultLine = "Seq Test exec time sec: " + (float)execTime / 1000;
+        LOG.info(resultLine);
+        return;
+      }
+      if (testType == TEST_TYPE_CLEANUP) {
+        cleanup(fs);
+        return;
+      }
+      createControlFile(fs, fileSize, nrFiles);
+      long tStart = System.currentTimeMillis();
+      if (testType == TEST_TYPE_WRITE)
+        writeTest(fs);
+      if (testType == TEST_TYPE_READ)
+        readTest(fs);
+      long execTime = System.currentTimeMillis() - tStart;
+    
+      analyzeResult(fs, testType, execTime, resFileName);
+    } catch(Exception e) {
+      System.err.print(StringUtils.stringifyException(e));
+      System.exit(-1);
+    }
+  }
+  
+  private static void analyzeResult( FileSystem fs, 
+                                     int testType,
+                                     long execTime,
+                                     String resFileName
+                                     ) throws IOException {
+    Path reduceFile;
+    if (testType == TEST_TYPE_WRITE)
+      reduceFile = new Path(WRITE_DIR, "part-00000");
+    else
+      reduceFile = new Path(READ_DIR, "part-00000");
+    DataInputStream in;
+    in = new DataInputStream(fs.open(reduceFile));
+  
+    BufferedReader lines;
+    lines = new BufferedReader(new InputStreamReader(in));
+    long tasks = 0;
+    long size = 0;
+    long time = 0;
+    float rate = 0;
+    float sqrate = 0;
+    String line;
+    while((line = lines.readLine()) != null) {
+      StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%");
+      String attr = tokens.nextToken(); 
+      if (attr.endsWith(":tasks"))
+        tasks = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith(":size"))
+        size = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith(":time"))
+        time = Long.parseLong(tokens.nextToken());
+      else if (attr.endsWith(":rate"))
+        rate = Float.parseFloat(tokens.nextToken());
+      else if (attr.endsWith(":sqrate"))
+        sqrate = Float.parseFloat(tokens.nextToken());
+    }
+    
+    double med = rate / 1000 / tasks;
+    double stdDev = Math.sqrt(Math.abs(sqrate / 1000 / tasks - med*med));
+    String resultLines[] = {
+      "----- TestDFSIO ----- : " + ((testType == TEST_TYPE_WRITE) ? "write" :
+                                    (testType == TEST_TYPE_READ) ? "read" : 
+                                    "unknown"),
+      "           Date & time: " + new Date(System.currentTimeMillis()),
+      "       Number of files: " + tasks,
+      "Total MBytes processed: " + size/MEGA,
+      "     Throughput mb/sec: " + size * 1000.0 / (time * MEGA),
+      "Average IO rate mb/sec: " + med,
+      " IO rate std deviation: " + stdDev,
+      "    Test exec time sec: " + (float)execTime / 1000,
+      "" };
+
+    PrintStream res = new PrintStream(
+                                      new FileOutputStream(
+                                                           new File(resFileName), true)); 
+    for(int i = 0; i < resultLines.length; i++) {
+      LOG.info(resultLines[i]);
+      res.println(resultLines[i]);
+    }
+  }
+
+  private static void cleanup(FileSystem fs) throws IOException {
+    LOG.info("Cleaning up test files");
+    fs.delete(new Path(TEST_ROOT_DIR), true);
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/TestFileSystem.java

@@ -0,0 +1,629 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.HashMap;
+import java.net.InetSocketAddress;
+import java.net.URI;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapred.lib.LongSumReducer;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+
+public class TestFileSystem extends TestCase {
+  private static final Log LOG = FileSystem.LOG;
+
+  private static Configuration conf = new Configuration();
+  private static int BUFFER_SIZE = conf.getInt("io.file.buffer.size", 4096);
+
+  private static final long MEGA = 1024 * 1024;
+  private static final int SEEKS_PER_FILE = 4;
+
+  private static String ROOT = System.getProperty("test.build.data","fs_test");
+  private static Path CONTROL_DIR = new Path(ROOT, "fs_control");
+  private static Path WRITE_DIR = new Path(ROOT, "fs_write");
+  private static Path READ_DIR = new Path(ROOT, "fs_read");
+  private static Path DATA_DIR = new Path(ROOT, "fs_data");
+
+  public void testFs() throws Exception {
+    testFs(10 * MEGA, 100, 0);
+  }
+
+  public static void testFs(long megaBytes, int numFiles, long seed)
+    throws Exception {
+
+    FileSystem fs = FileSystem.get(conf);
+
+    if (seed == 0)
+      seed = new Random().nextLong();
+
+    LOG.info("seed = "+seed);
+
+    createControlFile(fs, megaBytes, numFiles, seed);
+    writeTest(fs, false);
+    readTest(fs, false);
+    seekTest(fs, false);
+    fs.delete(CONTROL_DIR, true);
+    fs.delete(DATA_DIR, true);
+    fs.delete(WRITE_DIR, true);
+    fs.delete(READ_DIR, true);
+  }
+
+  public static void testCommandFormat() throws Exception {
+    // This should go to TestFsShell.java when it is added.
+    CommandFormat cf;
+    cf= new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
+    assertEquals(cf.parse(new String[] {"-get","file", "-"}, 1).get(1), "-");
+    assertEquals(cf.parse(new String[] {"-get","file","-ignoreCrc","/foo"}, 1).get(1),"/foo");
+    cf = new CommandFormat("tail", 1, 1, "f");
+    assertEquals(cf.parse(new String[] {"-tail","fileName"}, 1).get(0),"fileName");
+    assertEquals(cf.parse(new String[] {"-tail","-f","fileName"}, 1).get(0),"fileName");
+    cf = new CommandFormat("setrep", 2, 2, "R", "w");
+    assertEquals(cf.parse(new String[] {"-setrep","-R","2","/foo/bar"}, 1).get(1), "/foo/bar");
+    cf = new CommandFormat("put", 2, 10000);
+    assertEquals(cf.parse(new String[] {"-put", "-", "dest"}, 1).get(1), "dest"); 
+  }
+
+  public static void createControlFile(FileSystem fs,
+                                       long megaBytes, int numFiles,
+                                       long seed) throws Exception {
+
+    LOG.info("creating control file: "+megaBytes+" bytes, "+numFiles+" files");
+
+    Path controlFile = new Path(CONTROL_DIR, "files");
+    fs.delete(controlFile, true);
+    Random random = new Random(seed);
+
+    SequenceFile.Writer writer =
+      SequenceFile.createWriter(fs, conf, controlFile, 
+                                Text.class, LongWritable.class, CompressionType.NONE);
+
+    long totalSize = 0;
+    long maxSize = ((megaBytes / numFiles) * 2) + 1;
+    try {
+      while (totalSize < megaBytes) {
+        Text name = new Text(Long.toString(random.nextLong()));
+
+        long size = random.nextLong();
+        if (size < 0)
+          size = -size;
+        size = size % maxSize;
+
+        //LOG.info(" adding: name="+name+" size="+size);
+
+        writer.append(name, new LongWritable(size));
+
+        totalSize += size;
+      }
+    } finally {
+      writer.close();
+    }
+    LOG.info("created control file for: "+totalSize+" bytes");
+  }
+
+  public static class WriteMapper extends Configured
+      implements Mapper<Text, LongWritable, Text, LongWritable> {
+    
+    private Random random = new Random();
+    private byte[] buffer = new byte[BUFFER_SIZE];
+    private FileSystem fs;
+    private boolean fastCheck;
+
+    // a random suffix per task
+    private String suffix = "-"+random.nextLong();
+    
+    {
+      try {
+        fs = FileSystem.get(conf);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    public WriteMapper() { super(null); }
+    
+    public WriteMapper(Configuration conf) { super(conf); }
+
+    public void configure(JobConf job) {
+      setConf(job);
+      fastCheck = job.getBoolean("fs.test.fastCheck", false);
+    }
+
+    public void map(Text key, LongWritable value,
+                    OutputCollector<Text, LongWritable> collector,
+                    Reporter reporter)
+      throws IOException {
+      
+      String name = key.toString();
+      long size = value.get();
+      long seed = Long.parseLong(name);
+
+      random.setSeed(seed);
+      reporter.setStatus("creating " + name);
+
+      // write to temp file initially to permit parallel execution
+      Path tempFile = new Path(DATA_DIR, name+suffix);
+      OutputStream out = fs.create(tempFile);
+
+      long written = 0;
+      try {
+        while (written < size) {
+          if (fastCheck) {
+            Arrays.fill(buffer, (byte)random.nextInt(Byte.MAX_VALUE));
+          } else {
+            random.nextBytes(buffer);
+          }
+          long remains = size - written;
+          int length = (remains<=buffer.length) ? (int)remains : buffer.length;
+          out.write(buffer, 0, length);
+          written += length;
+          reporter.setStatus("writing "+name+"@"+written+"/"+size);
+        }
+      } finally {
+        out.close();
+      }
+      // rename to final location
+      fs.rename(tempFile, new Path(DATA_DIR, name));
+
+      collector.collect(new Text("bytes"), new LongWritable(written));
+
+      reporter.setStatus("wrote " + name);
+    }
+    
+    public void close() {
+    }
+    
+  }
+
+  public static void writeTest(FileSystem fs, boolean fastCheck)
+    throws Exception {
+
+    fs.delete(DATA_DIR, true);
+    fs.delete(WRITE_DIR, true);
+    
+    JobConf job = new JobConf(conf, TestFileSystem.class);
+    job.setBoolean("fs.test.fastCheck", fastCheck);
+
+    FileInputFormat.setInputPaths(job, CONTROL_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(WriteMapper.class);
+    job.setReducerClass(LongSumReducer.class);
+
+    FileOutputFormat.setOutputPath(job, WRITE_DIR);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(LongWritable.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+  public static class ReadMapper extends Configured
+      implements Mapper<Text, LongWritable, Text, LongWritable> {
+    
+    private Random random = new Random();
+    private byte[] buffer = new byte[BUFFER_SIZE];
+    private byte[] check  = new byte[BUFFER_SIZE];
+    private FileSystem fs;
+    private boolean fastCheck;
+
+    {
+      try {
+        fs = FileSystem.get(conf);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    public ReadMapper() { super(null); }
+    
+    public ReadMapper(Configuration conf) { super(conf); }
+
+    public void configure(JobConf job) {
+      setConf(job);
+      fastCheck = job.getBoolean("fs.test.fastCheck", false);
+    }
+
+    public void map(Text key, LongWritable value,
+                    OutputCollector<Text, LongWritable> collector,
+                    Reporter reporter)
+      throws IOException {
+      
+      String name = key.toString();
+      long size = value.get();
+      long seed = Long.parseLong(name);
+
+      random.setSeed(seed);
+      reporter.setStatus("opening " + name);
+
+      DataInputStream in =
+        new DataInputStream(fs.open(new Path(DATA_DIR, name)));
+
+      long read = 0;
+      try {
+        while (read < size) {
+          long remains = size - read;
+          int n = (remains<=buffer.length) ? (int)remains : buffer.length;
+          in.readFully(buffer, 0, n);
+          read += n;
+          if (fastCheck) {
+            Arrays.fill(check, (byte)random.nextInt(Byte.MAX_VALUE));
+          } else {
+            random.nextBytes(check);
+          }
+          if (n != buffer.length) {
+            Arrays.fill(buffer, n, buffer.length, (byte)0);
+            Arrays.fill(check, n, check.length, (byte)0);
+          }
+          assertTrue(Arrays.equals(buffer, check));
+
+          reporter.setStatus("reading "+name+"@"+read+"/"+size);
+
+        }
+      } finally {
+        in.close();
+      }
+
+      collector.collect(new Text("bytes"), new LongWritable(read));
+
+      reporter.setStatus("read " + name);
+    }
+    
+    public void close() {
+    }
+    
+  }
+
+  public static void readTest(FileSystem fs, boolean fastCheck)
+    throws Exception {
+
+    fs.delete(READ_DIR, true);
+
+    JobConf job = new JobConf(conf, TestFileSystem.class);
+    job.setBoolean("fs.test.fastCheck", fastCheck);
+
+
+    FileInputFormat.setInputPaths(job, CONTROL_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(ReadMapper.class);
+    job.setReducerClass(LongSumReducer.class);
+
+    FileOutputFormat.setOutputPath(job, READ_DIR);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(LongWritable.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+
+  public static class SeekMapper<K> extends Configured
+    implements Mapper<Text, LongWritable, K, LongWritable> {
+    
+    private Random random = new Random();
+    private byte[] check  = new byte[BUFFER_SIZE];
+    private FileSystem fs;
+    private boolean fastCheck;
+
+    {
+      try {
+        fs = FileSystem.get(conf);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    public SeekMapper() { super(null); }
+    
+    public SeekMapper(Configuration conf) { super(conf); }
+
+    public void configure(JobConf job) {
+      setConf(job);
+      fastCheck = job.getBoolean("fs.test.fastCheck", false);
+    }
+
+    public void map(Text key, LongWritable value,
+                    OutputCollector<K, LongWritable> collector,
+                    Reporter reporter)
+      throws IOException {
+      String name = key.toString();
+      long size = value.get();
+      long seed = Long.parseLong(name);
+
+      if (size == 0) return;
+
+      reporter.setStatus("opening " + name);
+
+      FSDataInputStream in = fs.open(new Path(DATA_DIR, name));
+        
+      try {
+        for (int i = 0; i < SEEKS_PER_FILE; i++) {
+          // generate a random position
+          long position = Math.abs(random.nextLong()) % size;
+          
+          // seek file to that position
+          reporter.setStatus("seeking " + name);
+          in.seek(position);
+          byte b = in.readByte();
+          
+          // check that byte matches
+          byte checkByte = 0;
+          // advance random state to that position
+          random.setSeed(seed);
+          for (int p = 0; p <= position; p+= check.length) {
+            reporter.setStatus("generating data for " + name);
+            if (fastCheck) {
+              checkByte = (byte)random.nextInt(Byte.MAX_VALUE);
+            } else {
+              random.nextBytes(check);
+              checkByte = check[(int)(position % check.length)];
+            }
+          }
+          assertEquals(b, checkByte);
+        }
+      } finally {
+        in.close();
+      }
+    }
+    
+    public void close() {
+    }
+    
+  }
+
+  public static void seekTest(FileSystem fs, boolean fastCheck)
+    throws Exception {
+
+    fs.delete(READ_DIR, true);
+
+    JobConf job = new JobConf(conf, TestFileSystem.class);
+    job.setBoolean("fs.test.fastCheck", fastCheck);
+
+    FileInputFormat.setInputPaths(job,CONTROL_DIR);
+    job.setInputFormat(SequenceFileInputFormat.class);
+
+    job.setMapperClass(SeekMapper.class);
+    job.setReducerClass(LongSumReducer.class);
+
+    FileOutputFormat.setOutputPath(job, READ_DIR);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(LongWritable.class);
+    job.setNumReduceTasks(1);
+    JobClient.runJob(job);
+  }
+
+
+  public static void main(String[] args) throws Exception {
+    int megaBytes = 10;
+    int files = 100;
+    boolean noRead = false;
+    boolean noWrite = false;
+    boolean noSeek = false;
+    boolean fastCheck = false;
+    long seed = new Random().nextLong();
+
+    String usage = "Usage: TestFileSystem -files N -megaBytes M [-noread] [-nowrite] [-noseek] [-fastcheck]";
+    
+    if (args.length == 0) {
+      System.err.println(usage);
+      System.exit(-1);
+    }
+    for (int i = 0; i < args.length; i++) {       // parse command line
+      if (args[i].equals("-files")) {
+        files = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-megaBytes")) {
+        megaBytes = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-noread")) {
+        noRead = true;
+      } else if (args[i].equals("-nowrite")) {
+        noWrite = true;
+      } else if (args[i].equals("-noseek")) {
+        noSeek = true;
+      } else if (args[i].equals("-fastcheck")) {
+        fastCheck = true;
+      }
+    }
+
+    LOG.info("seed = "+seed);
+    LOG.info("files = " + files);
+    LOG.info("megaBytes = " + megaBytes);
+  
+    FileSystem fs = FileSystem.get(conf);
+
+    if (!noWrite) {
+      createControlFile(fs, megaBytes*MEGA, files, seed);
+      writeTest(fs, fastCheck);
+    }
+    if (!noRead) {
+      readTest(fs, fastCheck);
+    }
+    if (!noSeek) {
+      seekTest(fs, fastCheck);
+    }
+  }
+
+  static Configuration createConf4Testing(String username) throws Exception {
+    Configuration conf = new Configuration();
+    UnixUserGroupInformation.saveToConf(conf,
+        UnixUserGroupInformation.UGI_PROPERTY_NAME,
+        new UnixUserGroupInformation(username, new String[]{"group"}));
+    return conf;    
+  }
+
+  public void testFsCache() throws Exception {
+    {
+      long now = System.currentTimeMillis();
+      Configuration[] conf = {new Configuration(),
+          createConf4Testing("foo" + now), createConf4Testing("bar" + now)};
+      FileSystem[] fs = new FileSystem[conf.length];
+  
+      for(int i = 0; i < conf.length; i++) {
+        fs[i] = FileSystem.get(conf[i]);
+        assertEquals(fs[i], FileSystem.get(conf[i]));
+        for(int j = 0; j < i; j++) {
+          assertFalse(fs[j] == fs[i]);
+        }
+      }
+      FileSystem.closeAll();
+    }
+    
+    {
+      try {
+        runTestCache(NameNode.DEFAULT_PORT);
+      } catch(java.net.BindException be) {
+        LOG.warn("Cannot test NameNode.DEFAULT_PORT (="
+            + NameNode.DEFAULT_PORT + ")", be);
+      }
+
+      runTestCache(0);
+    }
+  }
+  
+  static void runTestCache(int port) throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster(port, conf, 2, true, true, null, null);
+      URI uri = cluster.getFileSystem().getUri();
+      LOG.info("uri=" + uri);
+
+      {
+        FileSystem fs = FileSystem.get(uri, new Configuration());
+        checkPath(cluster, fs);
+        for(int i = 0; i < 100; i++) {
+          assertTrue(fs == FileSystem.get(uri, new Configuration()));
+        }
+      }
+      
+      if (port == NameNode.DEFAULT_PORT) {
+        //test explicit default port
+        URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(),
+            uri.getHost(), NameNode.DEFAULT_PORT, uri.getPath(),
+            uri.getQuery(), uri.getFragment());  
+        LOG.info("uri2=" + uri2);
+        FileSystem fs = FileSystem.get(uri2, conf);
+        checkPath(cluster, fs);
+        for(int i = 0; i < 100; i++) {
+          assertTrue(fs == FileSystem.get(uri2, new Configuration()));
+        }
+      }
+    } finally {
+      if (cluster != null) cluster.shutdown(); 
+    }
+  }
+  
+  static void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException {
+    InetSocketAddress add = cluster.getNameNode().getNameNodeAddress();
+    // Test upper/lower case
+    fileSys.checkPath(new Path("hdfs://" + add.getHostName().toUpperCase() + ":" + add.getPort()));
+  }
+
+  public void testFsClose() throws Exception {
+    {
+      Configuration conf = new Configuration();
+      new Path("file:///").getFileSystem(conf);
+      UnixUserGroupInformation.login(conf, true);
+      FileSystem.closeAll();
+    }
+
+    {
+      Configuration conf = new Configuration();
+      new Path("hftp://localhost:12345/").getFileSystem(conf);
+      UnixUserGroupInformation.login(conf, true);
+      FileSystem.closeAll();
+    }
+
+    {
+      Configuration conf = new Configuration();
+      FileSystem fs = new Path("hftp://localhost:12345/").getFileSystem(conf);
+      UnixUserGroupInformation.login(fs.getConf(), true);
+      FileSystem.closeAll();
+    }
+  }
+
+
+  public void testCacheKeysAreCaseInsensitive()
+    throws Exception
+  {
+    Configuration conf = new Configuration();
+    
+    // check basic equality
+    FileSystem.Cache.Key lowercaseCachekey1 = new FileSystem.Cache.Key(new URI("hftp://localhost:12345/"), conf);
+    FileSystem.Cache.Key lowercaseCachekey2 = new FileSystem.Cache.Key(new URI("hftp://localhost:12345/"), conf);
+    assertEquals( lowercaseCachekey1, lowercaseCachekey2 );
+
+    // check insensitive equality    
+    FileSystem.Cache.Key uppercaseCachekey = new FileSystem.Cache.Key(new URI("HFTP://Localhost:12345/"), conf);
+    assertEquals( lowercaseCachekey2, uppercaseCachekey );
+
+    // check behaviour with collections
+    List<FileSystem.Cache.Key> list = new ArrayList<FileSystem.Cache.Key>();
+    list.add(uppercaseCachekey);
+    assertTrue(list.contains(uppercaseCachekey));
+    assertTrue(list.contains(lowercaseCachekey2));
+
+    Set<FileSystem.Cache.Key> set = new HashSet<FileSystem.Cache.Key>();
+    set.add(uppercaseCachekey);
+    assertTrue(set.contains(uppercaseCachekey));
+    assertTrue(set.contains(lowercaseCachekey2));
+
+    Map<FileSystem.Cache.Key, String> map = new HashMap<FileSystem.Cache.Key, String>();
+    map.put(uppercaseCachekey, "");
+    assertTrue(map.containsKey(uppercaseCachekey));
+    assertTrue(map.containsKey(lowercaseCachekey2));    
+
+  }
+
+  public static void testFsUniqueness(long megaBytes, int numFiles, long seed)
+    throws Exception {
+
+    // multiple invocations of FileSystem.get return the same object.
+    FileSystem fs1 = FileSystem.get(conf);
+    FileSystem fs2 = FileSystem.get(conf);
+    assertTrue(fs1 == fs2);
+
+    // multiple invocations of FileSystem.newInstance return different objects
+    fs1 = FileSystem.newInstance(conf);
+    fs2 = FileSystem.newInstance(conf);
+    assertTrue(fs1 != fs2 && !fs1.equals(fs2));
+    fs1.close();
+    fs2.close();
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/fs/TestHarFileSystem.java

@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.tools.HadoopArchives;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * test the har file system
+ * create a har filesystem
+ * run fs commands
+ * and then run a map reduce job
+ */
+public class TestHarFileSystem extends TestCase {
+  private Path inputPath;
+  private MiniDFSCluster dfscluster;
+  private MiniMRCluster mapred;
+  private FileSystem fs;
+  private Path filea, fileb, filec;
+  private Path archivePath;
+  
+  protected void setUp() throws Exception {
+    super.setUp();
+    dfscluster = new MiniDFSCluster(new JobConf(), 2, true, null);
+    fs = dfscluster.getFileSystem();
+    mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
+    inputPath = new Path(fs.getHomeDirectory(), "test"); 
+    filea = new Path(inputPath,"a");
+    fileb = new Path(inputPath,"b");
+    filec = new Path(inputPath,"c");
+    archivePath = new Path(fs.getHomeDirectory(), "tmp");
+  }
+  
+  protected void tearDown() throws Exception {
+    try {
+      if (mapred != null) {
+        mapred.shutdown();
+      }
+      if (dfscluster != null) {
+        dfscluster.shutdown();
+      }
+    } catch(Exception e) {
+      System.err.println(e);
+    }
+    super.tearDown();
+  }
+  
+  static class TextMapperReducer implements Mapper<LongWritable, Text, Text, Text>, 
+            Reducer<Text, Text, Text, Text> {
+    
+    public void configure(JobConf conf) {
+      //do nothing 
+    }
+
+    public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
+      output.collect(value, new Text(""));
+    }
+
+    public void close() throws IOException {
+      // do nothing
+    }
+
+    public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
+      while(values.hasNext()) { 
+        values.next();
+        output.collect(key, null);
+      }
+    }
+  }
+  
+  public void testArchives() throws Exception {
+    fs.mkdirs(inputPath);
+    
+    FSDataOutputStream out = fs.create(filea); 
+    out.write("a".getBytes());
+    out.close();
+    out = fs.create(fileb);
+    out.write("b".getBytes());
+    out.close();
+    out = fs.create(filec);
+    out.write("c".getBytes());
+    out.close();
+    Configuration conf = mapred.createJobConf();
+    HadoopArchives har = new HadoopArchives(conf);
+    String[] args = new String[3];
+    //check for destination not specfied
+    args[0] = "-archiveName";
+    args[1] = "foo.har";
+    args[2] = inputPath.toString();
+    int ret = ToolRunner.run(har, args);
+    assertTrue(ret != 0);
+    args = new String[4];
+    //check for wrong archiveName
+    args[0] = "-archiveName";
+    args[1] = "/d/foo.har";
+    args[2] = inputPath.toString();
+    args[3] = archivePath.toString();
+    ret = ToolRunner.run(har, args);
+    assertTrue(ret != 0);
+//  se if dest is a file 
+    args[1] = "foo.har";
+    args[3] = filec.toString();
+    ret = ToolRunner.run(har, args);
+    assertTrue(ret != 0);
+    //this is a valid run
+    args[0] = "-archiveName";
+    args[1] = "foo.har";
+    args[2] = inputPath.toString();
+    args[3] = archivePath.toString();
+    ret = ToolRunner.run(har, args);
+    //checl for the existenece of the archive
+    assertTrue(ret == 0);
+    ///try running it again. it should not 
+    // override the directory
+    ret = ToolRunner.run(har, args);
+    assertTrue(ret != 0);
+    Path finalPath = new Path(archivePath, "foo.har");
+    Path fsPath = new Path(inputPath.toUri().getPath());
+    String relative = fsPath.toString().substring(1);
+    Path filePath = new Path(finalPath, relative);
+    //make it a har path 
+    Path harPath = new Path("har://" + filePath.toUri().getPath());
+    assertTrue(fs.exists(new Path(finalPath, "_index")));
+    assertTrue(fs.exists(new Path(finalPath, "_masterindex")));
+    assertTrue(!fs.exists(new Path(finalPath, "_logs")));
+    //creation tested
+    //check if the archive is same
+    // do ls and cat on all the files
+    FsShell shell = new FsShell(conf);
+    args = new String[2];
+    args[0] = "-ls";
+    args[1] = harPath.toString();
+    ret = ToolRunner.run(shell, args);
+    // ls should work.
+    assertTrue((ret == 0));
+    //now check for contents of filea
+    // fileb and filec
+    Path harFilea = new Path(harPath, "a");
+    Path harFileb = new Path(harPath, "b");
+    Path harFilec = new Path(harPath, "c");
+    FileSystem harFs = harFilea.getFileSystem(conf);
+    FSDataInputStream fin = harFs.open(harFilea);
+    byte[] b = new byte[4];
+    int readBytes = fin.read(b);
+    assertTrue("Empty read.", readBytes > 0);
+    fin.close();
+    assertTrue("strings are equal ", (b[0] == "a".getBytes()[0]));
+    fin = harFs.open(harFileb);
+    readBytes = fin.read(b);
+    assertTrue("Empty read.", readBytes > 0);
+    fin.close();
+    assertTrue("strings are equal ", (b[0] == "b".getBytes()[0]));
+    fin = harFs.open(harFilec);
+    readBytes = fin.read(b);
+    assertTrue("Empty read.", readBytes > 0);
+    fin.close();
+    assertTrue("strings are equal ", (b[0] == "c".getBytes()[0]));
+    // ok all files match 
+    // run a map reduce job
+    Path outdir = new Path(fs.getHomeDirectory(), "mapout"); 
+    JobConf jobconf = mapred.createJobConf();
+    FileInputFormat.addInputPath(jobconf, harPath);
+    jobconf.setInputFormat(TextInputFormat.class);
+    jobconf.setOutputFormat(TextOutputFormat.class);
+    FileOutputFormat.setOutputPath(jobconf, outdir);
+    jobconf.setMapperClass(TextMapperReducer.class);
+    jobconf.setMapOutputKeyClass(Text.class);
+    jobconf.setMapOutputValueClass(Text.class);
+    jobconf.setReducerClass(TextMapperReducer.class);
+    jobconf.setNumReduceTasks(1);
+    JobClient.runJob(jobconf);
+    args[1] = outdir.toString();
+    ret = ToolRunner.run(shell, args);
+    
+    FileStatus[] status = fs.globStatus(new Path(outdir, "part*"));
+    Path reduceFile = status[0].getPath();
+    FSDataInputStream reduceIn = fs.open(reduceFile);
+    b = new byte[6];
+    readBytes = reduceIn.read(b);
+    assertTrue("Should read 6 bytes.", readBytes == 6);
+    //assuming all the 6 bytes were read.
+    Text readTxt = new Text(b);
+    assertTrue("a\nb\nc\n".equals(readTxt.toString()));
+    assertTrue("number of bytes left should be -1", reduceIn.read(b) == -1);
+    reduceIn.close();
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/hdfs/NNBench.java

@@ -0,0 +1,964 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.Date;
+import java.io.DataInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.io.File;
+import java.io.BufferedReader;
+import java.util.StringTokenizer;
+import java.net.InetAddress;
+import java.text.SimpleDateFormat;
+import java.util.Iterator;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile;
+
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reducer;
+
+/**
+ * This program executes a specified operation that applies load to 
+ * the NameNode.
+ * 
+ * When run simultaneously on multiple nodes, this program functions 
+ * as a stress-test and benchmark for namenode, especially when 
+ * the number of bytes written to each file is small.
+ * 
+ * Valid operations are:
+ *   create_write
+ *   open_read
+ *   rename
+ *   delete
+ * 
+ * NOTE: The open_read, rename and delete operations assume that the files
+ *       they operate on are already available. The create_write operation 
+ *       must be run before running the other operations.
+ */
+
+public class NNBench {
+  private static final Log LOG = LogFactory.getLog(
+          "org.apache.hadoop.hdfs.NNBench");
+  
+  protected static String CONTROL_DIR_NAME = "control";
+  protected static String OUTPUT_DIR_NAME = "output";
+  protected static String DATA_DIR_NAME = "data";
+  protected static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log";
+  protected static final String NNBENCH_VERSION = "NameNode Benchmark 0.4";
+  
+  public static String operation = "none";
+  public static long numberOfMaps = 1l; // default is 1
+  public static long numberOfReduces = 1l; // default is 1
+  public static long startTime = 
+          System.currentTimeMillis() + (120 * 1000); // default is 'now' + 2min
+  public static long blockSize = 1l; // default is 1
+  public static int bytesToWrite = 0; // default is 0
+  public static long bytesPerChecksum = 1l; // default is 1
+  public static long numberOfFiles = 1l; // default is 1
+  public static short replicationFactorPerFile = 1; // default is 1
+  public static String baseDir = "/benchmarks/NNBench";  // default
+  public static boolean readFileAfterOpen = false; // default is to not read
+  
+  // Supported operations
+  private static final String OP_CREATE_WRITE = "create_write";
+  private static final String OP_OPEN_READ = "open_read";
+  private static final String OP_RENAME = "rename";
+  private static final String OP_DELETE = "delete";
+  
+  // To display in the format that matches the NN and DN log format
+  // Example: 2007-10-26 00:01:19,853
+  static SimpleDateFormat sdf = 
+          new SimpleDateFormat("yyyy-MM-dd' 'HH:mm:ss','S");
+
+  private static Configuration config = new Configuration();
+  
+  /**
+   * Clean up the files before a test run
+   * 
+   * @throws IOException on error
+   */
+  private static void cleanupBeforeTestrun() throws IOException {
+    FileSystem tempFS = FileSystem.get(config);
+    
+    // Delete the data directory only if it is the create/write operation
+    if (operation.equals(OP_CREATE_WRITE)) {
+      LOG.info("Deleting data directory");
+      tempFS.delete(new Path(baseDir, DATA_DIR_NAME), true);
+    }
+    tempFS.delete(new Path(baseDir, CONTROL_DIR_NAME), true);
+    tempFS.delete(new Path(baseDir, OUTPUT_DIR_NAME), true);
+  }
+  
+  /**
+   * Create control files before a test run.
+   * Number of files created is equal to the number of maps specified
+   * 
+   * @throws IOException on error
+   */
+  private static void createControlFiles() throws IOException {
+    FileSystem tempFS = FileSystem.get(config);
+    LOG.info("Creating " + numberOfMaps + " control files");
+
+    for (int i = 0; i < numberOfMaps; i++) {
+      String strFileName = "NNBench_Controlfile_" + i;
+      Path filePath = new Path(new Path(baseDir, CONTROL_DIR_NAME),
+              strFileName);
+
+      SequenceFile.Writer writer = null;
+      try {
+        writer = SequenceFile.createWriter(tempFS, config, filePath, Text.class, 
+                LongWritable.class, CompressionType.NONE);
+        writer.append(new Text(strFileName), new LongWritable(0l));
+      } catch(Exception e) {
+        throw new IOException(e.getLocalizedMessage());
+      } finally {
+        if (writer != null) {
+          writer.close();
+        }
+        writer = null;
+      }
+    }
+  }
+  /**
+   * Display version
+   */
+  private static void displayVersion() {
+    System.out.println(NNBENCH_VERSION);
+  }
+  
+  /**
+   * Display usage
+   */
+  private static void displayUsage() {
+    String usage =
+      "Usage: nnbench <options>\n" +
+      "Options:\n" +
+      "\t-operation <Available operations are " + OP_CREATE_WRITE + " " +
+      OP_OPEN_READ + " " + OP_RENAME + " " + OP_DELETE + ". " +
+      "This option is mandatory>\n" +
+      "\t * NOTE: The open_read, rename and delete operations assume " +
+      "that the files they operate on, are already available. " +
+      "The create_write operation must be run before running the " +
+      "other operations.\n" +
+      "\t-maps <number of maps. default is 1. This is not mandatory>\n" +
+      "\t-reduces <number of reduces. default is 1. This is not mandatory>\n" +
+      "\t-startTime <time to start, given in seconds from the epoch. " +
+      "Make sure this is far enough into the future, so all maps " +
+      "(operations) will start at the same time>. " +
+      "default is launch time + 2 mins. This is not mandatory \n" +
+      "\t-blockSize <Block size in bytes. default is 1. " + 
+      "This is not mandatory>\n" +
+      "\t-bytesToWrite <Bytes to write. default is 0. " + 
+      "This is not mandatory>\n" +
+      "\t-bytesPerChecksum <Bytes per checksum for the files. default is 1. " + 
+      "This is not mandatory>\n" +
+      "\t-numberOfFiles <number of files to create. default is 1. " +
+      "This is not mandatory>\n" +
+      "\t-replicationFactorPerFile <Replication factor for the files." +
+        " default is 1. This is not mandatory>\n" +
+      "\t-baseDir <base DFS path. default is /becnhmarks/NNBench. " +
+      "This is not mandatory>\n" +
+      "\t-readFileAfterOpen <true or false. if true, it reads the file and " +
+      "reports the average time to read. This is valid with the open_read " +
+      "operation. default is false. This is not mandatory>\n" +
+      "\t-help: Display the help statement\n";
+      
+    
+    System.out.println(usage);
+  }
+
+  /**
+   * check for arguments and fail if the values are not specified
+   */
+  public static void checkArgs(final int index, final int length) {
+    if (index == length) {
+      displayUsage();
+      System.exit(-1);
+    }
+  }
+  
+  /**
+   * Parse input arguments
+   * 
+   * @params args Command line inputs
+   */
+  public static void parseInputs(final String[] args) {
+    // If there are no command line arguments, exit
+    if (args.length == 0) {
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // Parse command line args
+    for (int i = 0; i < args.length; i++) {
+      if (args[i].equals("-operation")) {
+        operation = args[++i];
+      } else if (args[i].equals("-maps")) {
+        checkArgs(i + 1, args.length);
+        numberOfMaps = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-reduces")) {
+        checkArgs(i + 1, args.length);
+        numberOfReduces = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-startTime")) {
+        checkArgs(i + 1, args.length);
+        startTime = Long.parseLong(args[++i]) * 1000;
+      } else if (args[i].equals("-blockSize")) {
+        checkArgs(i + 1, args.length);
+        blockSize = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-bytesToWrite")) {
+        checkArgs(i + 1, args.length);
+        bytesToWrite = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-bytesPerChecksum")) {
+        checkArgs(i + 1, args.length);
+        bytesPerChecksum = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-numberOfFiles")) {
+        checkArgs(i + 1, args.length);
+        numberOfFiles = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-replicationFactorPerFile")) {
+        checkArgs(i + 1, args.length);
+        replicationFactorPerFile = Short.parseShort(args[++i]);
+      } else if (args[i].equals("-baseDir")) {
+        checkArgs(i + 1, args.length);
+        baseDir = args[++i];
+      } else if (args[i].equals("-readFileAfterOpen")) {
+        checkArgs(i + 1, args.length);
+        readFileAfterOpen = Boolean.parseBoolean(args[++i]);
+      } else if (args[i].equals("-help")) {
+        displayUsage();
+        System.exit(-1);
+      }
+    }
+    
+    LOG.info("Test Inputs: ");
+    LOG.info("           Test Operation: " + operation);
+    LOG.info("               Start time: " + sdf.format(new Date(startTime)));
+    LOG.info("           Number of maps: " + numberOfMaps);
+    LOG.info("        Number of reduces: " + numberOfReduces);
+    LOG.info("               Block Size: " + blockSize);
+    LOG.info("           Bytes to write: " + bytesToWrite);
+    LOG.info("       Bytes per checksum: " + bytesPerChecksum);
+    LOG.info("          Number of files: " + numberOfFiles);
+    LOG.info("       Replication factor: " + replicationFactorPerFile);
+    LOG.info("                 Base dir: " + baseDir);
+    LOG.info("     Read file after open: " + readFileAfterOpen);
+    
+    // Set user-defined parameters, so the map method can access the values
+    config.set("test.nnbench.operation", operation);
+    config.setLong("test.nnbench.maps", numberOfMaps);
+    config.setLong("test.nnbench.reduces", numberOfReduces);
+    config.setLong("test.nnbench.starttime", startTime);
+    config.setLong("test.nnbench.blocksize", blockSize);
+    config.setInt("test.nnbench.bytestowrite", bytesToWrite);
+    config.setLong("test.nnbench.bytesperchecksum", bytesPerChecksum);
+    config.setLong("test.nnbench.numberoffiles", numberOfFiles);
+    config.setInt("test.nnbench.replicationfactor", 
+            (int) replicationFactorPerFile);
+    config.set("test.nnbench.basedir", baseDir);
+    config.setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen);
+
+    config.set("test.nnbench.datadir.name", DATA_DIR_NAME);
+    config.set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME);
+    config.set("test.nnbench.controldir.name", CONTROL_DIR_NAME);
+  }
+  
+  /**
+   * Analyze the results
+   * 
+   * @throws IOException on error
+   */
+  private static void analyzeResults() throws IOException {
+    final FileSystem fs = FileSystem.get(config);
+    Path reduceFile = new Path(new Path(baseDir, OUTPUT_DIR_NAME),
+            "part-00000");
+
+    DataInputStream in;
+    in = new DataInputStream(fs.open(reduceFile));
+
+    BufferedReader lines;
+    lines = new BufferedReader(new InputStreamReader(in));
+
+    long totalTimeAL1 = 0l;
+    long totalTimeAL2 = 0l;
+    long totalTimeTPmS = 0l;
+    long lateMaps = 0l;
+    long numOfExceptions = 0l;
+    long successfulFileOps = 0l;
+    
+    long mapStartTimeTPmS = 0l;
+    long mapEndTimeTPmS = 0l;
+    
+    String resultTPSLine1 = null;
+    String resultTPSLine2 = null;
+    String resultALLine1 = null;
+    String resultALLine2 = null;
+    
+    String line;
+    while((line = lines.readLine()) != null) {
+      StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;");
+      String attr = tokens.nextToken();
+      if (attr.endsWith(":totalTimeAL1")) {
+        totalTimeAL1 = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":totalTimeAL2")) {
+        totalTimeAL2 = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":totalTimeTPmS")) {
+        totalTimeTPmS = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":latemaps")) {
+        lateMaps = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":numOfExceptions")) {
+        numOfExceptions = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":successfulFileOps")) {
+        successfulFileOps = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":mapStartTimeTPmS")) {
+        mapStartTimeTPmS = Long.parseLong(tokens.nextToken());
+      } else if (attr.endsWith(":mapEndTimeTPmS")) {
+        mapEndTimeTPmS = Long.parseLong(tokens.nextToken());
+      }
+    }
+    
+    // Average latency is the average time to perform 'n' number of
+    // operations, n being the number of files
+    double avgLatency1 = (double) totalTimeAL1 / (double) successfulFileOps;
+    double avgLatency2 = (double) totalTimeAL2 / (double) successfulFileOps;
+    
+    // The time it takes for the longest running map is measured. Using that,
+    // cluster transactions per second is calculated. It includes time to 
+    // retry any of the failed operations
+    double longestMapTimeTPmS = (double) (mapEndTimeTPmS - mapStartTimeTPmS);
+    double totalTimeTPS = (longestMapTimeTPmS == 0) ?
+            (1000 * successfulFileOps) :
+            (double) (1000 * successfulFileOps) / (double) longestMapTimeTPmS;
+            
+    // The time it takes to perform 'n' operations is calculated (in ms),
+    // n being the number of files. Using that time, the average execution 
+    // time is calculated. It includes time to retry any of the
+    // failed operations
+    double AverageExecutionTime = (totalTimeTPmS == 0) ?
+        (double) successfulFileOps : 
+        (double) (totalTimeTPmS / successfulFileOps);
+            
+    if (operation.equals(OP_CREATE_WRITE)) {
+      // For create/write/close, it is treated as two transactions,
+      // since a file create from a client perspective involves create and close
+      resultTPSLine1 = "               TPS: Create/Write/Close: " + 
+        (int) (totalTimeTPS * 2);
+      resultTPSLine2 = "Avg exec time (ms): Create/Write/Close: " + 
+        (double) AverageExecutionTime;
+      resultALLine1 = "            Avg Lat (ms): Create/Write: " + avgLatency1;
+      resultALLine2 = "                   Avg Lat (ms): Close: " + avgLatency2;
+    } else if (operation.equals(OP_OPEN_READ)) {
+      resultTPSLine1 = "                        TPS: Open/Read: " + 
+        (int) totalTimeTPS;
+      resultTPSLine2 = "         Avg Exec time (ms): Open/Read: " + 
+        (double) AverageExecutionTime;
+      resultALLine1 = "                    Avg Lat (ms): Open: " + avgLatency1;
+      if (readFileAfterOpen) {
+        resultALLine2 = "                  Avg Lat (ms): Read: " + avgLatency2;
+      }
+    } else if (operation.equals(OP_RENAME)) {
+      resultTPSLine1 = "                           TPS: Rename: " + 
+        (int) totalTimeTPS;
+      resultTPSLine2 = "            Avg Exec time (ms): Rename: " + 
+        (double) AverageExecutionTime;
+      resultALLine1 = "                  Avg Lat (ms): Rename: " + avgLatency1;
+    } else if (operation.equals(OP_DELETE)) {
+      resultTPSLine1 = "                           TPS: Delete: " + 
+        (int) totalTimeTPS;
+      resultTPSLine2 = "            Avg Exec time (ms): Delete: " + 
+        (double) AverageExecutionTime;
+      resultALLine1 = "                  Avg Lat (ms): Delete: " + avgLatency1;
+    }
+    
+    String resultLines[] = {
+    "-------------- NNBench -------------- : ",
+    "                               Version: " + NNBENCH_VERSION,
+    "                           Date & time: " + sdf.format(new Date(
+            System.currentTimeMillis())),
+    "",
+    "                        Test Operation: " + operation,
+    "                            Start time: " + 
+      sdf.format(new Date(startTime)),
+    "                           Maps to run: " + numberOfMaps,
+    "                        Reduces to run: " + numberOfReduces,
+    "                    Block Size (bytes): " + blockSize,
+    "                        Bytes to write: " + bytesToWrite,
+    "                    Bytes per checksum: " + bytesPerChecksum,
+    "                       Number of files: " + numberOfFiles,
+    "                    Replication factor: " + replicationFactorPerFile,
+    "            Successful file operations: " + successfulFileOps,
+    "",
+    "        # maps that missed the barrier: " + lateMaps,
+    "                          # exceptions: " + numOfExceptions,
+    "",
+    resultTPSLine1,
+    resultTPSLine2,
+    resultALLine1,
+    resultALLine2,
+    "",
+    "                 RAW DATA: AL Total #1: " + totalTimeAL1,
+    "                 RAW DATA: AL Total #2: " + totalTimeAL2,
+    "              RAW DATA: TPS Total (ms): " + totalTimeTPmS,
+    "       RAW DATA: Longest Map Time (ms): " + longestMapTimeTPmS,
+    "                   RAW DATA: Late maps: " + lateMaps,
+    "             RAW DATA: # of exceptions: " + numOfExceptions,
+    "" };
+
+    PrintStream res = new PrintStream(new FileOutputStream(
+            new File(DEFAULT_RES_FILE_NAME), true));
+    
+    // Write to a file and also dump to log
+    for(int i = 0; i < resultLines.length; i++) {
+      LOG.info(resultLines[i]);
+      res.println(resultLines[i]);
+    }
+  }
+  
+  /**
+   * Run the test
+   * 
+   * @throws IOException on error
+   */
+  public static void runTests() throws IOException {
+    config.setLong("io.bytes.per.checksum", bytesPerChecksum);
+    
+    JobConf job = new JobConf(config, NNBench.class);
+
+    job.setJobName("NNBench-" + operation);
+    FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME));
+    job.setInputFormat(SequenceFileInputFormat.class);
+    
+    // Explicitly set number of max map attempts to 1.
+    job.setMaxMapAttempts(1);
+    
+    // Explicitly turn off speculative execution
+    job.setSpeculativeExecution(false);
+
+    job.setMapperClass(NNBenchMapper.class);
+    job.setReducerClass(NNBenchReducer.class);
+
+    FileOutputFormat.setOutputPath(job, new Path(baseDir, OUTPUT_DIR_NAME));
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    job.setNumReduceTasks((int) numberOfReduces);
+    JobClient.runJob(job);
+  }
+  
+  /**
+   * Validate the inputs
+   */
+  public static void validateInputs() {
+    // If it is not one of the four operations, then fail
+    if (!operation.equals(OP_CREATE_WRITE) &&
+            !operation.equals(OP_OPEN_READ) &&
+            !operation.equals(OP_RENAME) &&
+            !operation.equals(OP_DELETE)) {
+      System.err.println("Error: Unknown operation: " + operation);
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If number of maps is a negative number, then fail
+    // Hadoop allows the number of maps to be 0
+    if (numberOfMaps < 0) {
+      System.err.println("Error: Number of maps must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If number of reduces is a negative number or 0, then fail
+    if (numberOfReduces <= 0) {
+      System.err.println("Error: Number of reduces must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+
+    // If blocksize is a negative number or 0, then fail
+    if (blockSize <= 0) {
+      System.err.println("Error: Block size must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If bytes to write is a negative number, then fail
+    if (bytesToWrite < 0) {
+      System.err.println("Error: Bytes to write must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If bytes per checksum is a negative number, then fail
+    if (bytesPerChecksum < 0) {
+      System.err.println("Error: Bytes per checksum must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If number of files is a negative number, then fail
+    if (numberOfFiles < 0) {
+      System.err.println("Error: Number of files must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If replication factor is a negative number, then fail
+    if (replicationFactorPerFile < 0) {
+      System.err.println("Error: Replication factor must be a positive number");
+      displayUsage();
+      System.exit(-1);
+    }
+    
+    // If block size is not a multiple of bytesperchecksum, fail
+    if (blockSize % bytesPerChecksum != 0) {
+      System.err.println("Error: Block Size in bytes must be a multiple of " +
+              "bytes per checksum: ");
+      displayUsage();
+      System.exit(-1);
+    }
+  }
+  /**
+  * Main method for running the NNBench benchmarks
+  *
+  * @throws IOException indicates a problem with test startup
+  */
+  public static void main(String[] args) throws IOException {
+    // Display the application version string
+    displayVersion();
+
+    // Parse the inputs
+    parseInputs(args);
+    
+    // Validate inputs
+    validateInputs();
+    
+    // Clean up files before the test run
+    cleanupBeforeTestrun();
+    
+    // Create control files before test run
+    createControlFiles();
+
+    // Run the tests as a map reduce job
+    runTests();
+    
+    // Analyze results
+    analyzeResults();
+  }
+
+  
+  /**
+   * Mapper class
+   */
+  static class NNBenchMapper extends Configured 
+          implements Mapper<Text, LongWritable, Text, Text> {
+    FileSystem filesystem = null;
+    private String hostName = null;
+
+    long numberOfFiles = 1l;
+    long blkSize = 1l;
+    short replFactor = 1;
+    int bytesToWrite = 0;
+    String baseDir = null;
+    String dataDirName = null;
+    String op = null;
+    boolean readFile = false;
+    final int MAX_OPERATION_EXCEPTIONS = 1000;
+    
+    // Data to collect from the operation
+    int numOfExceptions = 0;
+    long startTimeAL = 0l;
+    long totalTimeAL1 = 0l;
+    long totalTimeAL2 = 0l;
+    long successfulFileOps = 0l;
+    
+    /**
+     * Constructor
+     */
+    public NNBenchMapper() {
+    }
+    
+    /**
+     * Mapper base implementation
+     */
+    public void configure(JobConf conf) {
+      setConf(conf);
+      
+      try {
+        filesystem = FileSystem.get(conf);
+      } catch(Exception e) {
+        throw new RuntimeException("Cannot get file system.", e);
+      }
+      
+      try {
+        hostName = InetAddress.getLocalHost().getHostName();
+      } catch(Exception e) {
+        throw new RuntimeException("Error getting hostname", e);
+      }
+    }
+    
+    /**
+     * Mapper base implementation
+     */
+    public void close() throws IOException {
+    }
+    
+    /**
+    * Returns when the current number of seconds from the epoch equals
+    * the command line argument given by <code>-startTime</code>.
+    * This allows multiple instances of this program, running on clock
+    * synchronized nodes, to start at roughly the same time.
+    */
+    private boolean barrier() {
+      long startTime = getConf().getLong("test.nnbench.starttime", 0l);
+      long currentTime = System.currentTimeMillis();
+      long sleepTime = startTime - currentTime;
+      boolean retVal = false;
+      
+      // If the sleep time is greater than 0, then sleep and return
+      if (sleepTime > 0) {
+        LOG.info("Waiting in barrier for: " + sleepTime + " ms");
+      
+        try {
+          Thread.sleep(sleepTime);
+          retVal = true;
+        } catch (Exception e) {
+          retVal = false;
+        }
+      }
+      
+      return retVal;
+    }
+    
+    /**
+     * Map method
+     */ 
+    public void map(Text key, 
+            LongWritable value,
+            OutputCollector<Text, Text> output,
+            Reporter reporter) throws IOException {
+      Configuration conf = filesystem.getConf();
+      
+      numberOfFiles = conf.getLong("test.nnbench.numberoffiles", 1l);
+      blkSize = conf.getLong("test.nnbench.blocksize", 1l);
+      replFactor = (short) (conf.getInt("test.nnbench.replicationfactor", 1));
+      bytesToWrite = conf.getInt("test.nnbench.bytestowrite", 0);
+      baseDir = conf.get("test.nnbench.basedir");
+      dataDirName = conf.get("test.nnbench.datadir.name");
+      op = conf.get("test.nnbench.operation");
+      readFile = conf.getBoolean("test.nnbench.readFileAfterOpen", false);
+      
+      long totalTimeTPmS = 0l;
+      long startTimeTPmS = 0l;
+      long endTimeTPms = 0l;
+      
+      numOfExceptions = 0;
+      startTimeAL = 0l;
+      totalTimeAL1 = 0l;
+      totalTimeAL2 = 0l;
+      successfulFileOps = 0l;
+      
+      if (barrier()) {
+        if (op.equals(OP_CREATE_WRITE)) {
+          startTimeTPmS = System.currentTimeMillis();
+          doCreateWriteOp("file_" + hostName + "_", output, reporter);
+        } else if (op.equals(OP_OPEN_READ)) {
+          startTimeTPmS = System.currentTimeMillis();
+          doOpenReadOp("file_" + hostName + "_", output, reporter);
+        } else if (op.equals(OP_RENAME)) {
+          startTimeTPmS = System.currentTimeMillis();
+          doRenameOp("file_" + hostName + "_", output, reporter);
+        } else if (op.equals(OP_DELETE)) {
+          startTimeTPmS = System.currentTimeMillis();
+          doDeleteOp("file_" + hostName + "_", output, reporter);
+        }
+        
+        endTimeTPms = System.currentTimeMillis();
+        totalTimeTPmS = endTimeTPms - startTimeTPmS;
+      } else {
+        output.collect(new Text("l:latemaps"), new Text("1"));
+      }
+      
+      // collect after the map end time is measured
+      output.collect(new Text("l:totalTimeAL1"), 
+          new Text(String.valueOf(totalTimeAL1)));
+      output.collect(new Text("l:totalTimeAL2"), 
+          new Text(String.valueOf(totalTimeAL2)));
+      output.collect(new Text("l:numOfExceptions"), 
+          new Text(String.valueOf(numOfExceptions)));
+      output.collect(new Text("l:successfulFileOps"), 
+          new Text(String.valueOf(successfulFileOps)));
+      output.collect(new Text("l:totalTimeTPmS"), 
+              new Text(String.valueOf(totalTimeTPmS)));
+      output.collect(new Text("min:mapStartTimeTPmS"), 
+          new Text(String.valueOf(startTimeTPmS)));
+      output.collect(new Text("max:mapEndTimeTPmS"), 
+          new Text(String.valueOf(endTimeTPms)));
+    }
+    
+    /**
+     * Create and Write operation.
+     */
+    private void doCreateWriteOp(String name,
+            OutputCollector<Text, Text> output,
+            Reporter reporter) {
+      FSDataOutputStream out = null;
+      byte[] buffer = new byte[bytesToWrite];
+      
+      for (long l = 0l; l < numberOfFiles; l++) {
+        Path filePath = new Path(new Path(baseDir, dataDirName), 
+                name + "_" + l);
+
+        boolean successfulOp = false;
+        while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
+          try {
+            // Set up timer for measuring AL (transaction #1)
+            startTimeAL = System.currentTimeMillis();
+            // Create the file
+            // Use a buffer size of 512
+            out = filesystem.create(filePath, 
+                    true, 
+                    512, 
+                    replFactor, 
+                    blkSize);
+            out.write(buffer);
+            totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
+
+            // Close the file / file output stream
+            // Set up timers for measuring AL (transaction #2)
+            startTimeAL = System.currentTimeMillis();
+            out.close();
+            
+            totalTimeAL2 += (System.currentTimeMillis() - startTimeAL);
+            successfulOp = true;
+            successfulFileOps ++;
+
+            reporter.setStatus("Finish "+ l + " files");
+          } catch (IOException e) {
+            LOG.info("Exception recorded in op: " +
+                    "Create/Write/Close");
+ 
+            numOfExceptions++;
+          }
+        }
+      }
+    }
+    
+    /**
+     * Open operation
+     */
+    private void doOpenReadOp(String name,
+            OutputCollector<Text, Text> output,
+            Reporter reporter) {
+      FSDataInputStream input = null;
+      byte[] buffer = new byte[bytesToWrite];
+      
+      for (long l = 0l; l < numberOfFiles; l++) {
+        Path filePath = new Path(new Path(baseDir, dataDirName), 
+                name + "_" + l);
+
+        boolean successfulOp = false;
+        while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
+          try {
+            // Set up timer for measuring AL
+            startTimeAL = System.currentTimeMillis();
+            input = filesystem.open(filePath);
+            totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
+            
+            // If the file needs to be read (specified at command line)
+            if (readFile) {
+              startTimeAL = System.currentTimeMillis();
+              input.readFully(buffer);
+
+              totalTimeAL2 += (System.currentTimeMillis() - startTimeAL);
+            }
+            input.close();
+            successfulOp = true;
+            successfulFileOps ++;
+
+            reporter.setStatus("Finish "+ l + " files");
+          } catch (IOException e) {
+            LOG.info("Exception recorded in op: OpenRead " + e);
+            numOfExceptions++;
+          }
+        }
+      }
+    }
+    
+    /**
+     * Rename operation
+     */
+    private void doRenameOp(String name,
+            OutputCollector<Text, Text> output,
+            Reporter reporter) {
+      for (long l = 0l; l < numberOfFiles; l++) {
+        Path filePath = new Path(new Path(baseDir, dataDirName), 
+                name + "_" + l);
+        Path filePathR = new Path(new Path(baseDir, dataDirName), 
+                name + "_r_" + l);
+
+        boolean successfulOp = false;
+        while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
+          try {
+            // Set up timer for measuring AL
+            startTimeAL = System.currentTimeMillis();
+            filesystem.rename(filePath, filePathR);
+            totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
+            
+            successfulOp = true;
+            successfulFileOps ++;
+
+            reporter.setStatus("Finish "+ l + " files");
+          } catch (IOException e) {
+            LOG.info("Exception recorded in op: Rename");
+
+            numOfExceptions++;
+          }
+        }
+      }
+    }
+    
+    /**
+     * Delete operation
+     */
+    private void doDeleteOp(String name,
+            OutputCollector<Text, Text> output,
+            Reporter reporter) {
+      for (long l = 0l; l < numberOfFiles; l++) {
+        Path filePath = new Path(new Path(baseDir, dataDirName), 
+                name + "_" + l);
+        
+        boolean successfulOp = false;
+        while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
+          try {
+            // Set up timer for measuring AL
+            startTimeAL = System.currentTimeMillis();
+            filesystem.delete(filePath, true);
+            totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
+            
+            successfulOp = true;
+            successfulFileOps ++;
+
+            reporter.setStatus("Finish "+ l + " files");
+          } catch (IOException e) {
+            LOG.info("Exception in recorded op: Delete");
+
+            numOfExceptions++;
+          }
+        }
+      }
+    }
+  }
+  
+  /**
+   * Reducer class
+   */
+  static class NNBenchReducer extends MapReduceBase
+      implements Reducer<Text, Text, Text, Text> {
+
+    protected String hostName;
+
+    public NNBenchReducer () {
+      LOG.info("Starting NNBenchReducer !!!");
+      try {
+        hostName = java.net.InetAddress.getLocalHost().getHostName();
+      } catch(Exception e) {
+        hostName = "localhost";
+      }
+      LOG.info("Starting NNBenchReducer on " + hostName);
+    }
+
+    /**
+     * Reduce method
+     */
+    public void reduce(Text key, 
+                       Iterator<Text> values,
+                       OutputCollector<Text, Text> output, 
+                       Reporter reporter
+                       ) throws IOException {
+      String field = key.toString();
+      
+      reporter.setStatus("starting " + field + " ::host = " + hostName);
+      
+      // sum long values
+      if (field.startsWith("l:")) {
+        long lSum = 0;
+        while (values.hasNext()) {
+          lSum += Long.parseLong(values.next().toString());
+        }
+        output.collect(key, new Text(String.valueOf(lSum)));
+      }
+      
+      if (field.startsWith("min:")) {
+        long minVal = -1;
+        while (values.hasNext()) {
+          long value = Long.parseLong(values.next().toString());
+          
+          if (minVal == -1) {
+            minVal = value;
+          } else {
+            if (value != 0 && value < minVal) {
+              minVal = value;
+            }
+          }
+        }
+        output.collect(key, new Text(String.valueOf(minVal)));
+      }
+      
+      if (field.startsWith("max:")) {
+        long maxVal = -1;
+        while (values.hasNext()) {
+          long value = Long.parseLong(values.next().toString());
+          
+          if (maxVal == -1) {
+            maxVal = value;
+          } else {
+            if (value > maxVal) {
+              maxVal = value;
+            }
+          }
+        }
+        output.collect(key, new Text(String.valueOf(maxVal)));
+      }
+      
+      reporter.setStatus("finished " + field + " ::host = " + hostName);
+    }
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/hdfs/NNBenchWithoutMR.java

@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * This program executes a specified operation that applies load to 
+ * the NameNode. Possible operations include create/writing files,
+ * opening/reading files, renaming files, and deleting files.
+ * 
+ * When run simultaneously on multiple nodes, this program functions 
+ * as a stress-test and benchmark for namenode, especially when 
+ * the number of bytes written to each file is small.
+ * 
+ * This version does not use the map reduce framework
+ * 
+ */
+public class NNBenchWithoutMR {
+  
+  private static final Log LOG = LogFactory.getLog(
+                                            "org.apache.hadoop.hdfs.NNBench");
+  
+  // variable initialzed from command line arguments
+  private static long startTime = 0;
+  private static int numFiles = 0;
+  private static long bytesPerBlock = 1;
+  private static long blocksPerFile = 0;
+  private static long bytesPerFile = 1;
+  private static Path baseDir = null;
+    
+  // variables initialized in main()
+  private static FileSystem fileSys = null;
+  private static Path taskDir = null;
+  private static String uniqueId = null;
+  private static byte[] buffer;
+  private static long maxExceptionsPerFile = 200;
+    
+  /**
+   * Returns when the current number of seconds from the epoch equals
+   * the command line argument given by <code>-startTime</code>.
+   * This allows multiple instances of this program, running on clock
+   * synchronized nodes, to start at roughly the same time.
+   */
+  static void barrier() {
+    long sleepTime;
+    while ((sleepTime = startTime - System.currentTimeMillis()) > 0) {
+      try {
+        Thread.sleep(sleepTime);
+      } catch (InterruptedException ex) {
+      }
+    }
+  }
+    
+  static private void handleException(String operation, Throwable e, 
+                                      int singleFileExceptions) {
+    LOG.warn("Exception while " + operation + ": " +
+             StringUtils.stringifyException(e));
+    if (singleFileExceptions >= maxExceptionsPerFile) {
+      throw new RuntimeException(singleFileExceptions + 
+        " exceptions for a single file exceeds threshold. Aborting");
+    }
+  }
+  
+  /**
+   * Create and write to a given number of files.  Repeat each remote
+   * operation until is suceeds (does not throw an exception).
+   *
+   * @return the number of exceptions caught
+   */
+  static int createWrite() {
+    int totalExceptions = 0;
+    FSDataOutputStream out = null;
+    boolean success = false;
+    for (int index = 0; index < numFiles; index++) {
+      int singleFileExceptions = 0;
+      do { // create file until is succeeds or max exceptions reached
+        try {
+          out = fileSys.create(
+                               new Path(taskDir, "" + index), false, 512, (short)1, bytesPerBlock);
+          success = true;
+        } catch (IOException ioe) { 
+          success=false; 
+          totalExceptions++;
+          handleException("creating file #" + index, ioe, ++singleFileExceptions);
+        }
+      } while (!success);
+      long toBeWritten = bytesPerFile;
+      while (toBeWritten > 0) {
+        int nbytes = (int) Math.min(buffer.length, toBeWritten);
+        toBeWritten -= nbytes;
+        try { // only try once
+          out.write(buffer, 0, nbytes);
+        } catch (IOException ioe) {
+          totalExceptions++;
+          handleException("writing to file #" + index, ioe, ++singleFileExceptions);
+        }
+      }
+      do { // close file until is succeeds
+        try {
+          out.close();
+          success = true;
+        } catch (IOException ioe) {
+          success=false; 
+          totalExceptions++;
+          handleException("closing file #" + index, ioe, ++singleFileExceptions);
+        }
+      } while (!success);
+    }
+    return totalExceptions;
+  }
+    
+  /**
+   * Open and read a given number of files.
+   *
+   * @return the number of exceptions caught
+   */
+  static int openRead() {
+    int totalExceptions = 0;
+    FSDataInputStream in = null;
+    for (int index = 0; index < numFiles; index++) {
+      int singleFileExceptions = 0;
+      try {
+        in = fileSys.open(new Path(taskDir, "" + index), 512);
+        long toBeRead = bytesPerFile;
+        while (toBeRead > 0) {
+          int nbytes = (int) Math.min(buffer.length, toBeRead);
+          toBeRead -= nbytes;
+          try { // only try once
+            in.read(buffer, 0, nbytes);
+          } catch (IOException ioe) {
+            totalExceptions++;
+            handleException("reading from file #" + index, ioe, ++singleFileExceptions);
+          }
+        }
+        in.close();
+      } catch (IOException ioe) { 
+        totalExceptions++;
+        handleException("opening file #" + index, ioe, ++singleFileExceptions);
+      }
+    }
+    return totalExceptions;
+  }
+    
+  /**
+   * Rename a given number of files.  Repeat each remote
+   * operation until is suceeds (does not throw an exception).
+   *
+   * @return the number of exceptions caught
+   */
+  static int rename() {
+    int totalExceptions = 0;
+    boolean success = false;
+    for (int index = 0; index < numFiles; index++) {
+      int singleFileExceptions = 0;
+      do { // rename file until is succeeds
+        try {
+          boolean result = fileSys.rename(
+                                          new Path(taskDir, "" + index), new Path(taskDir, "A" + index));
+          success = true;
+        } catch (IOException ioe) { 
+          success=false; 
+          totalExceptions++;
+          handleException("creating file #" + index, ioe, ++singleFileExceptions);
+       }
+      } while (!success);
+    }
+    return totalExceptions;
+  }
+    
+  /**
+   * Delete a given number of files.  Repeat each remote
+   * operation until is suceeds (does not throw an exception).
+   *
+   * @return the number of exceptions caught
+   */
+  static int delete() {
+    int totalExceptions = 0;
+    boolean success = false;
+    for (int index = 0; index < numFiles; index++) {
+      int singleFileExceptions = 0;
+      do { // delete file until is succeeds
+        try {
+          boolean result = fileSys.delete(new Path(taskDir, "A" + index), true);
+          success = true;
+        } catch (IOException ioe) { 
+          success=false; 
+          totalExceptions++;
+          handleException("creating file #" + index, ioe, ++singleFileExceptions);
+        }
+      } while (!success);
+    }
+    return totalExceptions;
+  }
+    
+  /**
+   * This launches a given namenode operation (<code>-operation</code>),
+   * starting at a given time (<code>-startTime</code>).  The files used
+   * by the openRead, rename, and delete operations are the same files
+   * created by the createWrite operation.  Typically, the program
+   * would be run four times, once for each operation in this order:
+   * createWrite, openRead, rename, delete.
+   *
+   * <pre>
+   * Usage: nnbench 
+   *          -operation <one of createWrite, openRead, rename, or delete>
+   *          -baseDir <base output/input DFS path>
+   *          -startTime <time to start, given in seconds from the epoch>
+   *          -numFiles <number of files to create, read, rename, or delete>
+   *          -blocksPerFile <number of blocks to create per file>
+   *         [-bytesPerBlock <number of bytes to write to each block, default is 1>]
+   *         [-bytesPerChecksum <value for io.bytes.per.checksum>]
+   * </pre>
+   *
+   * @throws IOException indicates a problem with test startup
+   */
+  public static void main(String[] args) throws IOException {
+    String version = "NameNodeBenchmark.0.3";
+    System.out.println(version);
+    int bytesPerChecksum = -1;
+    
+    String usage =
+      "Usage: nnbench " +
+      "  -operation <one of createWrite, openRead, rename, or delete> " +
+      "  -baseDir <base output/input DFS path> " +
+      "  -startTime <time to start, given in seconds from the epoch> " +
+      "  -numFiles <number of files to create> " +
+      "  -blocksPerFile <number of blocks to create per file> " +
+      "  [-bytesPerBlock <number of bytes to write to each block, default is 1>] " +
+      "  [-bytesPerChecksum <value for io.bytes.per.checksum>]" +
+      "Note: bytesPerBlock MUST be a multiple of bytesPerChecksum";
+    
+    String operation = null;
+    for (int i = 0; i < args.length; i++) { // parse command line
+      if (args[i].equals("-baseDir")) {
+        baseDir = new Path(args[++i]);
+      } else if (args[i].equals("-numFiles")) {
+        numFiles = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-blocksPerFile")) {
+        blocksPerFile = Integer.parseInt(args[++i]);
+      } else if (args[i].equals("-bytesPerBlock")) {
+        bytesPerBlock = Long.parseLong(args[++i]);
+      } else if (args[i].equals("-bytesPerChecksum")) {
+        bytesPerChecksum = Integer.parseInt(args[++i]);        
+      } else if (args[i].equals("-startTime")) {
+        startTime = Long.parseLong(args[++i]) * 1000;
+      } else if (args[i].equals("-operation")) {
+        operation = args[++i];
+      } else {
+        System.out.println(usage);
+        System.exit(-1);
+      }
+    }
+    bytesPerFile = bytesPerBlock * blocksPerFile;
+    
+    JobConf jobConf = new JobConf(new Configuration(), NNBench.class);
+    
+    if ( bytesPerChecksum < 0 ) { // if it is not set in cmdline
+      bytesPerChecksum = jobConf.getInt("io.bytes.per.checksum", 512);
+    }
+    jobConf.set("io.bytes.per.checksum", Integer.toString(bytesPerChecksum));
+    
+    System.out.println("Inputs: ");
+    System.out.println("   operation: " + operation);
+    System.out.println("   baseDir: " + baseDir);
+    System.out.println("   startTime: " + startTime);
+    System.out.println("   numFiles: " + numFiles);
+    System.out.println("   blocksPerFile: " + blocksPerFile);
+    System.out.println("   bytesPerBlock: " + bytesPerBlock);
+    System.out.println("   bytesPerChecksum: " + bytesPerChecksum);
+    
+    if (operation == null ||  // verify args
+        baseDir == null ||
+        numFiles < 1 ||
+        blocksPerFile < 1 ||
+        bytesPerBlock < 0 ||
+        bytesPerBlock % bytesPerChecksum != 0)
+      {
+        System.err.println(usage);
+        System.exit(-1);
+      }
+    
+    fileSys = FileSystem.get(jobConf);
+    uniqueId = java.net.InetAddress.getLocalHost().getHostName();
+    taskDir = new Path(baseDir, uniqueId);
+    // initialize buffer used for writing/reading file
+    buffer = new byte[(int) Math.min(bytesPerFile, 32768L)];
+    
+    Date execTime;
+    Date endTime;
+    long duration;
+    int exceptions = 0;
+    barrier(); // wait for coordinated start time
+    execTime = new Date();
+    System.out.println("Job started: " + startTime);
+    if (operation.equals("createWrite")) {
+      if (!fileSys.mkdirs(taskDir)) {
+        throw new IOException("Mkdirs failed to create " + taskDir.toString());
+      }
+      exceptions = createWrite();
+    } else if (operation.equals("openRead")) {
+      exceptions = openRead();
+    } else if (operation.equals("rename")) {
+      exceptions = rename();
+    } else if (operation.equals("delete")) {
+      exceptions = delete();
+    } else {
+      System.err.println(usage);
+      System.exit(-1);
+    }
+    endTime = new Date();
+    System.out.println("Job ended: " + endTime);
+    duration = (endTime.getTime() - execTime.getTime()) /1000;
+    System.out.println("The " + operation + " job took " + duration + " seconds.");
+    System.out.println("The job recorded " + exceptions + " exceptions.");
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/io/FileBench.java

@@ -0,0 +1,603 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class FileBench extends Configured implements Tool {
+
+  static int printUsage() {
+    ToolRunner.printGenericCommandUsage(System.out);
+    System.out.println(
+"Usage: Task list:           -[no]r -[no]w\n" +
+"       Format:              -[no]seq -[no]txt\n" +
+"       CompressionCodec:    -[no]zip -[no]pln\n" +
+"       CompressionType:     -[no]blk -[no]rec\n" +
+"       Required:            -dir <working dir>\n" +
+"All valid combinations are implicitly enabled, unless an option is enabled\n" +
+"explicitly. For example, specifying \"-zip\", excludes -pln,\n" +
+"unless they are also explicitly included, as in \"-pln -zip\"\n" +
+"Note that CompressionType params only apply to SequenceFiles\n\n" +
+"Useful options to set:\n" +
+"-D fs.default.name=\"file:///\" \\\n" +
+"-D fs.file.impl=org.apache.hadoop.fs.RawLocalFileSystem \\\n" +
+"-D filebench.file.bytes=$((10*1024*1024*1024)) \\\n" +
+"-D filebench.key.words=5 \\\n" +
+"-D filebench.val.words=20\n");
+    return -1;
+  }
+
+  static String[] keys;
+  static String[] values;
+  static StringBuilder sentence = new StringBuilder();
+
+  private static String generateSentence(Random r, int noWords) {
+    sentence.setLength(0);
+    for (int i=0; i < noWords; ++i) {
+      sentence.append(words[r.nextInt(words.length)]);
+      sentence.append(" ");
+    }
+    return sentence.toString();
+  }
+
+  // fill keys, values with ~1.5 blocks for block-compressed seq fill
+  private static void fillBlocks(JobConf conf) {
+    Random r = new Random();
+    long seed = conf.getLong("filebench.seed", -1);
+    if (seed > 0) {
+      r.setSeed(seed);
+    }
+
+    int keylen = conf.getInt("filebench.key.words", 5);
+    int vallen = conf.getInt("filebench.val.words", 20);
+    int acc = (3 * conf.getInt("io.seqfile.compress.blocksize", 1000000)) >> 1;
+    ArrayList<String> k = new ArrayList<String>();
+    ArrayList<String> v = new ArrayList<String>();
+    for (int i = 0; acc > 0; ++i) {
+      String s = generateSentence(r, keylen);
+      acc -= s.length();
+      k.add(s);
+      s = generateSentence(r, vallen);
+      acc -= s.length();
+      v.add(s);
+    }
+    keys = k.toArray(new String[0]);
+    values = v.toArray(new String[0]);
+  }
+
+  @SuppressWarnings("unchecked") // OutputFormat instantiation
+  static long writeBench(JobConf conf) throws IOException {
+    long filelen = conf.getLong("filebench.file.bytes", 5 * 1024 * 1024 * 1024);
+    Text key = new Text();
+    Text val = new Text();
+
+    final String fn = conf.get("test.filebench.name", "");
+    final Path outd = FileOutputFormat.getOutputPath(conf);
+    conf.set("mapred.work.output.dir", outd.toString());
+    OutputFormat outf = conf.getOutputFormat();
+    RecordWriter<Text,Text> rw =
+      outf.getRecordWriter(outd.getFileSystem(conf), conf, fn,
+                           Reporter.NULL);
+    try {
+      long acc = 0L;
+      Date start = new Date();
+      for (int i = 0; acc < filelen; ++i) {
+        i %= keys.length;
+        key.set(keys[i]);
+        val.set(values[i]);
+        rw.write(key, val);
+        acc += keys[i].length();
+        acc += values[i].length();
+      }
+      Date end = new Date();
+      return end.getTime() - start.getTime();
+    } finally {
+      rw.close(Reporter.NULL);
+    }
+  }
+
+  @SuppressWarnings("unchecked") // InputFormat instantiation
+  static long readBench(JobConf conf) throws IOException {
+    InputFormat inf = conf.getInputFormat();
+    final String fn = conf.get("test.filebench.name", "");
+    Path pin = new Path(FileInputFormat.getInputPaths(conf)[0], fn);
+    FileStatus in = pin.getFileSystem(conf).getFileStatus(pin);
+    RecordReader rr = inf.getRecordReader(new FileSplit(pin, 0, in.getLen(), 
+                                          (String[])null), conf, Reporter.NULL);
+    try {
+      Object key = rr.createKey();
+      Object val = rr.createValue();
+      Date start = new Date();
+      while (rr.next(key, val));
+      Date end = new Date();
+      return end.getTime() - start.getTime();
+    } finally {
+      rr.close();
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new Configuration(), new FileBench(), args);
+    System.exit(res);
+  }
+
+  /**
+   * Process params from command line and run set of benchmarks specified.
+   */
+  public int run(String[] argv) throws IOException {
+    JobConf job = new JobConf(getConf());
+    EnumSet<CCodec> cc = null;
+    EnumSet<CType> ct = null;
+    EnumSet<Format> f = null;
+    EnumSet<RW> rw = null;
+    Path root = null;
+    FileSystem fs = FileSystem.get(job);
+    for(int i = 0; i < argv.length; ++i) {
+      try {
+        if ("-dir".equals(argv[i])) {
+          root = new Path(argv[++i]).makeQualified(fs);
+          System.out.println("DIR: " + root.toString());
+        } else if ("-seed".equals(argv[i])) {
+          job.setLong("filebench.seed", Long.valueOf(argv[++i]));
+        } else if (argv[i].startsWith("-no")) {
+          String arg = argv[i].substring(3);
+          cc = rem(CCodec.class, cc, arg);
+          ct = rem(CType.class, ct, arg);
+          f =  rem(Format.class, f, arg);
+          rw = rem(RW.class, rw, arg);
+        } else {
+          String arg = argv[i].substring(1);
+          cc = add(CCodec.class, cc, arg);
+          ct = add(CType.class, ct, arg);
+          f =  add(Format.class, f, arg);
+          rw = add(RW.class, rw, arg);
+        }
+      } catch (Exception e) {
+        throw (IOException)new IOException().initCause(e);
+      }
+    }
+    if (null == root) {
+      System.out.println("Missing -dir param");
+      printUsage();
+      return -1;
+    }
+
+    fillBlocks(job);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(Text.class);
+    FileInputFormat.setInputPaths(job, root);
+    FileOutputFormat.setOutputPath(job, root);
+
+    if (null == cc) cc = EnumSet.allOf(CCodec.class);
+    if (null == ct) ct = EnumSet.allOf(CType.class);
+    if (null == f)  f  = EnumSet.allOf(Format.class);
+    if (null == rw) rw = EnumSet.allOf(RW.class);
+    for (RW rwop : rw) {
+      for (Format fmt : f) {
+        fmt.configure(job);
+        for (CCodec cod : cc) {
+          cod.configure(job);
+          if (!(fmt == Format.txt || cod == CCodec.pln)) {
+            for (CType typ : ct) {
+              String fn =
+                fmt.name().toUpperCase() + "_" +
+                cod.name().toUpperCase() + "_" +
+                typ.name().toUpperCase();
+              typ.configure(job);
+              System.out.print(rwop.name().toUpperCase() + " " + fn + ": ");
+              System.out.println(rwop.exec(fn, job) / 1000 +
+                  " seconds");
+            }
+          } else {
+            String fn =
+              fmt.name().toUpperCase() + "_" +
+              cod.name().toUpperCase();
+            Path p = new Path(root, fn);
+            if (rwop == RW.r && !fs.exists(p)) {
+              fn += cod.getExt();
+            }
+            System.out.print(rwop.name().toUpperCase() + " " + fn + ": ");
+            System.out.println(rwop.exec(fn, job) / 1000 +
+                " seconds");
+          }
+        }
+      }
+    }
+    return 0;
+  }
+
+  // overwrought argument processing and wordlist follow
+  enum CCodec {
+    zip(GzipCodec.class, ".gz"), pln(null, "");
+
+    Class<? extends CompressionCodec> inf;
+    String ext;
+    CCodec(Class<? extends CompressionCodec> inf, String ext) {
+      this.inf = inf;
+      this.ext = ext;
+    }
+    public void configure(JobConf job) {
+      if (inf != null) {
+        job.setBoolean("mapred.output.compress", true);
+        job.setClass("mapred.output.compression.codec", inf,
+            CompressionCodec.class);
+      } else {
+        job.setBoolean("mapred.output.compress", false);
+      }
+    }
+    public String getExt() { return ext; }
+  }
+  enum CType {
+    blk("BLOCK"),
+    rec("RECORD");
+
+    String typ;
+    CType(String typ) { this.typ = typ; }
+    public void configure(JobConf job) {
+      job.set("mapred.map.output.compression.type", typ);
+      job.set("mapred.output.compression.type", typ);
+    }
+  }
+  enum Format {
+    seq(SequenceFileInputFormat.class, SequenceFileOutputFormat.class),
+    txt(TextInputFormat.class, TextOutputFormat.class);
+
+    Class<? extends InputFormat> inf;
+    Class<? extends OutputFormat> of;
+    Format(Class<? extends InputFormat> inf, Class<? extends OutputFormat> of) {
+      this.inf = inf;
+      this.of = of;
+    }
+    public void configure(JobConf job) {
+      if (null != inf) job.setInputFormat(inf);
+      if (null != of) job.setOutputFormat(of);
+    }
+  }
+  enum RW {
+    w() {
+      public long exec(String fn, JobConf job) throws IOException {
+        job.set("test.filebench.name", fn);
+        return writeBench(job);
+      }
+    },
+
+    r() {
+      public long exec(String fn, JobConf job) throws IOException {
+        job.set("test.filebench.name", fn);
+        return readBench(job);
+      }
+    };
+
+    public abstract long exec(String fn, JobConf job) throws IOException;
+  }
+  static Map<Class<? extends Enum>, Map<String,? extends Enum>> fullmap
+    = new HashMap<Class<? extends Enum>, Map<String,? extends Enum>>();
+  static {
+    // can't effectively use Enum::valueOf
+    Map<String,CCodec> m1 = new HashMap<String,CCodec>();
+    for (CCodec v : CCodec.values()) m1.put(v.name(), v);
+    fullmap.put(CCodec.class, m1);
+    Map<String,CType> m2 = new HashMap<String,CType>();
+    for (CType v : CType.values()) m2.put(v.name(), v);
+    fullmap.put(CType.class, m2);
+    Map<String,Format> m3 = new HashMap<String,Format>();
+    for (Format v : Format.values()) m3.put(v.name(), v);
+    fullmap.put(Format.class, m3);
+    Map<String,RW> m4 = new HashMap<String,RW>();
+    for (RW v : RW.values()) m4.put(v.name(), v);
+    fullmap.put(RW.class, m4);
+  }
+
+  public static <T extends Enum<T>> EnumSet<T> rem(Class<T> c,
+      EnumSet<T> set, String s) {
+    if (null != fullmap.get(c) && fullmap.get(c).get(s) != null) {
+      if (null == set) {
+        set = EnumSet.allOf(c);
+      }
+      set.remove(fullmap.get(c).get(s));
+    }
+    return set;
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T extends Enum<T>> EnumSet<T> add(Class<T> c,
+      EnumSet<T> set, String s) {
+    if (null != fullmap.get(c) && fullmap.get(c).get(s) != null) {
+      if (null == set) {
+        set = EnumSet.noneOf(c);
+      }
+      set.add((T)fullmap.get(c).get(s));
+    }
+    return set;
+  }
+
+  /**
+   * A random list of 1000 words from /usr/share/dict/words
+   */
+  private static final String[] words = {
+    "diurnalness", "Homoiousian", "spiranthic", "tetragynian",
+    "silverhead", "ungreat", "lithograph", "exploiter",
+    "physiologian", "by", "hellbender", "Filipendula",
+    "undeterring", "antiscolic", "pentagamist", "hypoid",
+    "cacuminal", "sertularian", "schoolmasterism", "nonuple",
+    "gallybeggar", "phytonic", "swearingly", "nebular",
+    "Confervales", "thermochemically", "characinoid", "cocksuredom",
+    "fallacious", "feasibleness", "debromination", "playfellowship",
+    "tramplike", "testa", "participatingly", "unaccessible",
+    "bromate", "experientialist", "roughcast", "docimastical",
+    "choralcelo", "blightbird", "peptonate", "sombreroed",
+    "unschematized", "antiabolitionist", "besagne", "mastication",
+    "bromic", "sviatonosite", "cattimandoo", "metaphrastical",
+    "endotheliomyoma", "hysterolysis", "unfulminated", "Hester",
+    "oblongly", "blurredness", "authorling", "chasmy",
+    "Scorpaenidae", "toxihaemia", "Dictograph", "Quakerishly",
+    "deaf", "timbermonger", "strammel", "Thraupidae",
+    "seditious", "plerome", "Arneb", "eristically",
+    "serpentinic", "glaumrie", "socioromantic", "apocalypst",
+    "tartrous", "Bassaris", "angiolymphoma", "horsefly",
+    "kenno", "astronomize", "euphemious", "arsenide",
+    "untongued", "parabolicness", "uvanite", "helpless",
+    "gemmeous", "stormy", "templar", "erythrodextrin",
+    "comism", "interfraternal", "preparative", "parastas",
+    "frontoorbital", "Ophiosaurus", "diopside", "serosanguineous",
+    "ununiformly", "karyological", "collegian", "allotropic",
+    "depravity", "amylogenesis", "reformatory", "epidymides",
+    "pleurotropous", "trillium", "dastardliness", "coadvice",
+    "embryotic", "benthonic", "pomiferous", "figureheadship",
+    "Megaluridae", "Harpa", "frenal", "commotion",
+    "abthainry", "cobeliever", "manilla", "spiciferous",
+    "nativeness", "obispo", "monilioid", "biopsic",
+    "valvula", "enterostomy", "planosubulate", "pterostigma",
+    "lifter", "triradiated", "venialness", "tum",
+    "archistome", "tautness", "unswanlike", "antivenin",
+    "Lentibulariaceae", "Triphora", "angiopathy", "anta",
+    "Dawsonia", "becomma", "Yannigan", "winterproof",
+    "antalgol", "harr", "underogating", "ineunt",
+    "cornberry", "flippantness", "scyphostoma", "approbation",
+    "Ghent", "Macraucheniidae", "scabbiness", "unanatomized",
+    "photoelasticity", "eurythermal", "enation", "prepavement",
+    "flushgate", "subsequentially", "Edo", "antihero",
+    "Isokontae", "unforkedness", "porriginous", "daytime",
+    "nonexecutive", "trisilicic", "morphiomania", "paranephros",
+    "botchedly", "impugnation", "Dodecatheon", "obolus",
+    "unburnt", "provedore", "Aktistetae", "superindifference",
+    "Alethea", "Joachimite", "cyanophilous", "chorograph",
+    "brooky", "figured", "periclitation", "quintette",
+    "hondo", "ornithodelphous", "unefficient", "pondside",
+    "bogydom", "laurinoxylon", "Shiah", "unharmed",
+    "cartful", "noncrystallized", "abusiveness", "cromlech",
+    "japanned", "rizzomed", "underskin", "adscendent",
+    "allectory", "gelatinousness", "volcano", "uncompromisingly",
+    "cubit", "idiotize", "unfurbelowed", "undinted",
+    "magnetooptics", "Savitar", "diwata", "ramosopalmate",
+    "Pishquow", "tomorn", "apopenptic", "Haversian",
+    "Hysterocarpus", "ten", "outhue", "Bertat",
+    "mechanist", "asparaginic", "velaric", "tonsure",
+    "bubble", "Pyrales", "regardful", "glyphography",
+    "calabazilla", "shellworker", "stradametrical", "havoc",
+    "theologicopolitical", "sawdust", "diatomaceous", "jajman",
+    "temporomastoid", "Serrifera", "Ochnaceae", "aspersor",
+    "trailmaking", "Bishareen", "digitule", "octogynous",
+    "epididymitis", "smokefarthings", "bacillite", "overcrown",
+    "mangonism", "sirrah", "undecorated", "psychofugal",
+    "bismuthiferous", "rechar", "Lemuridae", "frameable",
+    "thiodiazole", "Scanic", "sportswomanship", "interruptedness",
+    "admissory", "osteopaedion", "tingly", "tomorrowness",
+    "ethnocracy", "trabecular", "vitally", "fossilism",
+    "adz", "metopon", "prefatorial", "expiscate",
+    "diathermacy", "chronist", "nigh", "generalizable",
+    "hysterogen", "aurothiosulphuric", "whitlowwort", "downthrust",
+    "Protestantize", "monander", "Itea", "chronographic",
+    "silicize", "Dunlop", "eer", "componental",
+    "spot", "pamphlet", "antineuritic", "paradisean",
+    "interruptor", "debellator", "overcultured", "Florissant",
+    "hyocholic", "pneumatotherapy", "tailoress", "rave",
+    "unpeople", "Sebastian", "thermanesthesia", "Coniferae",
+    "swacking", "posterishness", "ethmopalatal", "whittle",
+    "analgize", "scabbardless", "naught", "symbiogenetically",
+    "trip", "parodist", "columniform", "trunnel",
+    "yawler", "goodwill", "pseudohalogen", "swangy",
+    "cervisial", "mediateness", "genii", "imprescribable",
+    "pony", "consumptional", "carposporangial", "poleax",
+    "bestill", "subfebrile", "sapphiric", "arrowworm",
+    "qualminess", "ultraobscure", "thorite", "Fouquieria",
+    "Bermudian", "prescriber", "elemicin", "warlike",
+    "semiangle", "rotular", "misthread", "returnability",
+    "seraphism", "precostal", "quarried", "Babylonism",
+    "sangaree", "seelful", "placatory", "pachydermous",
+    "bozal", "galbulus", "spermaphyte", "cumbrousness",
+    "pope", "signifier", "Endomycetaceae", "shallowish",
+    "sequacity", "periarthritis", "bathysphere", "pentosuria",
+    "Dadaism", "spookdom", "Consolamentum", "afterpressure",
+    "mutter", "louse", "ovoviviparous", "corbel",
+    "metastoma", "biventer", "Hydrangea", "hogmace",
+    "seizing", "nonsuppressed", "oratorize", "uncarefully",
+    "benzothiofuran", "penult", "balanocele", "macropterous",
+    "dishpan", "marten", "absvolt", "jirble",
+    "parmelioid", "airfreighter", "acocotl", "archesporial",
+    "hypoplastral", "preoral", "quailberry", "cinque",
+    "terrestrially", "stroking", "limpet", "moodishness",
+    "canicule", "archididascalian", "pompiloid", "overstaid",
+    "introducer", "Italical", "Christianopaganism", "prescriptible",
+    "subofficer", "danseuse", "cloy", "saguran",
+    "frictionlessly", "deindividualization", "Bulanda", "ventricous",
+    "subfoliar", "basto", "scapuloradial", "suspend",
+    "stiffish", "Sphenodontidae", "eternal", "verbid",
+    "mammonish", "upcushion", "barkometer", "concretion",
+    "preagitate", "incomprehensible", "tristich", "visceral",
+    "hemimelus", "patroller", "stentorophonic", "pinulus",
+    "kerykeion", "brutism", "monstership", "merciful",
+    "overinstruct", "defensibly", "bettermost", "splenauxe",
+    "Mormyrus", "unreprimanded", "taver", "ell",
+    "proacquittal", "infestation", "overwoven", "Lincolnlike",
+    "chacona", "Tamil", "classificational", "lebensraum",
+    "reeveland", "intuition", "Whilkut", "focaloid",
+    "Eleusinian", "micromembrane", "byroad", "nonrepetition",
+    "bacterioblast", "brag", "ribaldrous", "phytoma",
+    "counteralliance", "pelvimetry", "pelf", "relaster",
+    "thermoresistant", "aneurism", "molossic", "euphonym",
+    "upswell", "ladhood", "phallaceous", "inertly",
+    "gunshop", "stereotypography", "laryngic", "refasten",
+    "twinling", "oflete", "hepatorrhaphy", "electrotechnics",
+    "cockal", "guitarist", "topsail", "Cimmerianism",
+    "larklike", "Llandovery", "pyrocatechol", "immatchable",
+    "chooser", "metrocratic", "craglike", "quadrennial",
+    "nonpoisonous", "undercolored", "knob", "ultratense",
+    "balladmonger", "slait", "sialadenitis", "bucketer",
+    "magnificently", "unstipulated", "unscourged", "unsupercilious",
+    "packsack", "pansophism", "soorkee", "percent",
+    "subirrigate", "champer", "metapolitics", "spherulitic",
+    "involatile", "metaphonical", "stachyuraceous", "speckedness",
+    "bespin", "proboscidiform", "gul", "squit",
+    "yeelaman", "peristeropode", "opacousness", "shibuichi",
+    "retinize", "yote", "misexposition", "devilwise",
+    "pumpkinification", "vinny", "bonze", "glossing",
+    "decardinalize", "transcortical", "serphoid", "deepmost",
+    "guanajuatite", "wemless", "arval", "lammy",
+    "Effie", "Saponaria", "tetrahedral", "prolificy",
+    "excerpt", "dunkadoo", "Spencerism", "insatiately",
+    "Gilaki", "oratorship", "arduousness", "unbashfulness",
+    "Pithecolobium", "unisexuality", "veterinarian", "detractive",
+    "liquidity", "acidophile", "proauction", "sural",
+    "totaquina", "Vichyite", "uninhabitedness", "allegedly",
+    "Gothish", "manny", "Inger", "flutist",
+    "ticktick", "Ludgatian", "homotransplant", "orthopedical",
+    "diminutively", "monogoneutic", "Kenipsim", "sarcologist",
+    "drome", "stronghearted", "Fameuse", "Swaziland",
+    "alen", "chilblain", "beatable", "agglomeratic",
+    "constitutor", "tendomucoid", "porencephalous", "arteriasis",
+    "boser", "tantivy", "rede", "lineamental",
+    "uncontradictableness", "homeotypical", "masa", "folious",
+    "dosseret", "neurodegenerative", "subtransverse", "Chiasmodontidae",
+    "palaeotheriodont", "unstressedly", "chalcites", "piquantness",
+    "lampyrine", "Aplacentalia", "projecting", "elastivity",
+    "isopelletierin", "bladderwort", "strander", "almud",
+    "iniquitously", "theologal", "bugre", "chargeably",
+    "imperceptivity", "meriquinoidal", "mesophyte", "divinator",
+    "perfunctory", "counterappellant", "synovial", "charioteer",
+    "crystallographical", "comprovincial", "infrastapedial", "pleasurehood",
+    "inventurous", "ultrasystematic", "subangulated", "supraoesophageal",
+    "Vaishnavism", "transude", "chrysochrous", "ungrave",
+    "reconciliable", "uninterpleaded", "erlking", "wherefrom",
+    "aprosopia", "antiadiaphorist", "metoxazine", "incalculable",
+    "umbellic", "predebit", "foursquare", "unimmortal",
+    "nonmanufacture", "slangy", "predisputant", "familist",
+    "preaffiliate", "friarhood", "corelysis", "zoonitic",
+    "halloo", "paunchy", "neuromimesis", "aconitine",
+    "hackneyed", "unfeeble", "cubby", "autoschediastical",
+    "naprapath", "lyrebird", "inexistency", "leucophoenicite",
+    "ferrogoslarite", "reperuse", "uncombable", "tambo",
+    "propodiale", "diplomatize", "Russifier", "clanned",
+    "corona", "michigan", "nonutilitarian", "transcorporeal",
+    "bought", "Cercosporella", "stapedius", "glandularly",
+    "pictorially", "weism", "disilane", "rainproof",
+    "Caphtor", "scrubbed", "oinomancy", "pseudoxanthine",
+    "nonlustrous", "redesertion", "Oryzorictinae", "gala",
+    "Mycogone", "reappreciate", "cyanoguanidine", "seeingness",
+    "breadwinner", "noreast", "furacious", "epauliere",
+    "omniscribent", "Passiflorales", "uninductive", "inductivity",
+    "Orbitolina", "Semecarpus", "migrainoid", "steprelationship",
+    "phlogisticate", "mesymnion", "sloped", "edificator",
+    "beneficent", "culm", "paleornithology", "unurban",
+    "throbless", "amplexifoliate", "sesquiquintile", "sapience",
+    "astucious", "dithery", "boor", "ambitus",
+    "scotching", "uloid", "uncompromisingness", "hoove",
+    "waird", "marshiness", "Jerusalem", "mericarp",
+    "unevoked", "benzoperoxide", "outguess", "pyxie",
+    "hymnic", "euphemize", "mendacity", "erythremia",
+    "rosaniline", "unchatteled", "lienteria", "Bushongo",
+    "dialoguer", "unrepealably", "rivethead", "antideflation",
+    "vinegarish", "manganosiderite", "doubtingness", "ovopyriform",
+    "Cephalodiscus", "Muscicapa", "Animalivora", "angina",
+    "planispheric", "ipomoein", "cuproiodargyrite", "sandbox",
+    "scrat", "Munnopsidae", "shola", "pentafid",
+    "overstudiousness", "times", "nonprofession", "appetible",
+    "valvulotomy", "goladar", "uniarticular", "oxyterpene",
+    "unlapsing", "omega", "trophonema", "seminonflammable",
+    "circumzenithal", "starer", "depthwise", "liberatress",
+    "unleavened", "unrevolting", "groundneedle", "topline",
+    "wandoo", "umangite", "ordinant", "unachievable",
+    "oversand", "snare", "avengeful", "unexplicit",
+    "mustafina", "sonable", "rehabilitative", "eulogization",
+    "papery", "technopsychology", "impressor", "cresylite",
+    "entame", "transudatory", "scotale", "pachydermatoid",
+    "imaginary", "yeat", "slipped", "stewardship",
+    "adatom", "cockstone", "skyshine", "heavenful",
+    "comparability", "exprobratory", "dermorhynchous", "parquet",
+    "cretaceous", "vesperal", "raphis", "undangered",
+    "Glecoma", "engrain", "counteractively", "Zuludom",
+    "orchiocatabasis", "Auriculariales", "warriorwise", "extraorganismal",
+    "overbuilt", "alveolite", "tetchy", "terrificness",
+    "widdle", "unpremonished", "rebilling", "sequestrum",
+    "equiconvex", "heliocentricism", "catabaptist", "okonite",
+    "propheticism", "helminthagogic", "calycular", "giantly",
+    "wingable", "golem", "unprovided", "commandingness",
+    "greave", "haply", "doina", "depressingly",
+    "subdentate", "impairment", "decidable", "neurotrophic",
+    "unpredict", "bicorporeal", "pendulant", "flatman",
+    "intrabred", "toplike", "Prosobranchiata", "farrantly",
+    "toxoplasmosis", "gorilloid", "dipsomaniacal", "aquiline",
+    "atlantite", "ascitic", "perculsive", "prospectiveness",
+    "saponaceous", "centrifugalization", "dinical", "infravaginal",
+    "beadroll", "affaite", "Helvidian", "tickleproof",
+    "abstractionism", "enhedge", "outwealth", "overcontribute",
+    "coldfinch", "gymnastic", "Pincian", "Munychian",
+    "codisjunct", "quad", "coracomandibular", "phoenicochroite",
+    "amender", "selectivity", "putative", "semantician",
+    "lophotrichic", "Spatangoidea", "saccharogenic", "inferent",
+    "Triconodonta", "arrendation", "sheepskin", "taurocolla",
+    "bunghole", "Machiavel", "triakistetrahedral", "dehairer",
+    "prezygapophysial", "cylindric", "pneumonalgia", "sleigher",
+    "emir", "Socraticism", "licitness", "massedly",
+    "instructiveness", "sturdied", "redecrease", "starosta",
+    "evictor", "orgiastic", "squdge", "meloplasty",
+    "Tsonecan", "repealableness", "swoony", "myesthesia",
+    "molecule", "autobiographist", "reciprocation", "refective",
+    "unobservantness", "tricae", "ungouged", "floatability",
+    "Mesua", "fetlocked", "chordacentrum", "sedentariness",
+    "various", "laubanite", "nectopod", "zenick",
+    "sequentially", "analgic", "biodynamics", "posttraumatic",
+    "nummi", "pyroacetic", "bot", "redescend",
+    "dispermy", "undiffusive", "circular", "trillion",
+    "Uraniidae", "ploration", "discipular", "potentness",
+    "sud", "Hu", "Eryon", "plugger",
+    "subdrainage", "jharal", "abscission", "supermarket",
+    "countergabion", "glacierist", "lithotresis", "minniebush",
+    "zanyism", "eucalypteol", "sterilely", "unrealize",
+    "unpatched", "hypochondriacism", "critically", "cheesecutter",
+  };
+}

src/test/hdfs-with-mr/org/apache/hadoop/io/TestSequenceFileMergeProgress.java

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator;
+import org.apache.hadoop.io.SequenceFile.Sorter.SegmentDescriptor;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.mapred.*;
+
+import junit.framework.TestCase;
+import org.apache.commons.logging.*;
+
+public class TestSequenceFileMergeProgress extends TestCase {
+  private static final Log LOG = FileInputFormat.LOG;
+  private static final int RECORDS = 10000;
+  
+  public void testMergeProgressWithNoCompression() throws IOException {
+    runTest(SequenceFile.CompressionType.NONE);
+  }
+
+  public void testMergeProgressWithRecordCompression() throws IOException {
+    runTest(SequenceFile.CompressionType.RECORD);
+  }
+
+  public void testMergeProgressWithBlockCompression() throws IOException {
+    runTest(SequenceFile.CompressionType.BLOCK);
+  }
+
+  public void runTest(CompressionType compressionType) throws IOException {
+    JobConf job = new JobConf();
+    FileSystem fs = FileSystem.getLocal(job);
+    Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
+    Path file = new Path(dir, "test.seq");
+    Path tempDir = new Path(dir, "tmp");
+
+    fs.delete(dir, true);
+    FileInputFormat.setInputPaths(job, dir);
+    fs.mkdirs(tempDir);
+
+    LongWritable tkey = new LongWritable();
+    Text tval = new Text();
+
+    SequenceFile.Writer writer =
+      SequenceFile.createWriter(fs, job, file, LongWritable.class, Text.class,
+        compressionType, new DefaultCodec());
+    try {
+      for (int i = 0; i < RECORDS; ++i) {
+        tkey.set(1234);
+        tval.set("valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue");
+        writer.append(tkey, tval);
+      }
+    } finally {
+      writer.close();
+    }
+    
+    long fileLength = fs.getFileStatus(file).getLen();
+    LOG.info("With compression = " + compressionType + ": "
+        + "compressed length = " + fileLength);
+    
+    SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, 
+        job.getOutputKeyComparator(), job.getMapOutputKeyClass(),
+        job.getMapOutputValueClass(), job);
+    Path[] paths = new Path[] {file};
+    RawKeyValueIterator rIter = sorter.merge(paths, tempDir, false);
+    int count = 0;
+    while (rIter.next()) {
+      count++;
+    }
+    assertEquals(RECORDS, count);
+    assertEquals(1.0f, rIter.getProgress().get());
+  }
+
+}

src/test/hdfs-with-mr/org/apache/hadoop/ipc/TestSocketFactory.java

@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.net.StandardSocketFactory;
+
+/**
+ * This class checks that RPCs can use specialized socket factories.
+ */
+public class TestSocketFactory extends TestCase {
+
+  /**
+   * Check that we can reach a NameNode or a JobTracker using a specific
+   * socket factory
+   */
+  public void testSocketFactory() throws IOException {
+    // Create a standard mini-cluster
+    Configuration sconf = new Configuration();
+    MiniDFSCluster cluster = new MiniDFSCluster(sconf, 1, true, null);
+    final int nameNodePort = cluster.getNameNodePort();
+
+    // Get a reference to its DFS directly
+    FileSystem fs = cluster.getFileSystem();
+    assertTrue(fs instanceof DistributedFileSystem);
+    DistributedFileSystem directDfs = (DistributedFileSystem) fs;
+
+    // Get another reference via network using a specific socket factory
+    Configuration cconf = new Configuration();
+    FileSystem.setDefaultUri(cconf, String.format("hdfs://localhost:%s/",
+        nameNodePort + 10));
+    cconf.set("hadoop.rpc.socket.factory.class.default",
+        "org.apache.hadoop.ipc.DummySocketFactory");
+    cconf.set("hadoop.rpc.socket.factory.class.ClientProtocol",
+        "org.apache.hadoop.ipc.DummySocketFactory");
+    cconf.set("hadoop.rpc.socket.factory.class.JobSubmissionProtocol",
+        "org.apache.hadoop.ipc.DummySocketFactory");
+
+    fs = FileSystem.get(cconf);
+    assertTrue(fs instanceof DistributedFileSystem);
+    DistributedFileSystem dfs = (DistributedFileSystem) fs;
+
+    JobClient client = null;
+    MiniMRCluster mr = null;
+    try {
+      // This will test RPC to the NameNode only.
+      // could we test Client-DataNode connections?
+      Path filePath = new Path("/dir");
+
+      assertFalse(directDfs.exists(filePath));
+      assertFalse(dfs.exists(filePath));
+
+      directDfs.mkdirs(filePath);
+      assertTrue(directDfs.exists(filePath));
+      assertTrue(dfs.exists(filePath));
+
+      // This will test TPC to a JobTracker
+      fs = FileSystem.get(sconf);
+      mr = new MiniMRCluster(1, fs.getUri().toString(), 1);
+      final int jobTrackerPort = mr.getJobTrackerPort();
+
+      JobConf jconf = new JobConf(cconf);
+      jconf.set("mapred.job.tracker", String.format("localhost:%d",
+          jobTrackerPort + 10));
+      client = new JobClient(jconf);
+
+      JobStatus[] jobs = client.jobsToComplete();
+      assertTrue(jobs.length == 0);
+
+    } finally {
+      try {
+        if (client != null)
+          client.close();
+      } catch (Exception ignored) {
+        // nothing we can do
+        ignored.printStackTrace();
+      }
+      try {
+        if (dfs != null)
+          dfs.close();
+
+      } catch (Exception ignored) {
+        // nothing we can do
+        ignored.printStackTrace();
+      }
+      try {
+        if (directDfs != null)
+          directDfs.close();
+
+      } catch (Exception ignored) {
+        // nothing we can do
+        ignored.printStackTrace();
+      }
+      try {
+        if (cluster != null)
+          cluster.shutdown();
+
+      } catch (Exception ignored) {
+        // nothing we can do
+        ignored.printStackTrace();
+      }
+      if (mr != null) {
+        try {
+          mr.shutdown();
+        } catch (Exception ignored) {
+          ignored.printStackTrace();
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Dummy socket factory which shift TPC ports by subtracting 10 when
+ * establishing a connection
+ */
+class DummySocketFactory extends StandardSocketFactory {
+  /**
+   * Default empty constructor (for use with the reflection API).
+   */
+  public DummySocketFactory() {
+  }
+
+  /* @inheritDoc */
+  @Override
+  public Socket createSocket() throws IOException {
+    return new Socket() {
+      @Override
+      public void connect(SocketAddress addr, int timeout)
+          throws IOException {
+
+        assert (addr instanceof InetSocketAddress);
+        InetSocketAddress iaddr = (InetSocketAddress) addr;
+        SocketAddress newAddr = null;
+        if (iaddr.isUnresolved())
+          newAddr =
+              new InetSocketAddress(iaddr.getHostName(),
+                  iaddr.getPort() - 10);
+        else
+          newAddr =
+              new InetSocketAddress(iaddr.getAddress(), iaddr.getPort() - 10);
+        System.out.printf("Test socket: rerouting %s to %s\n", iaddr,
+            newAddr);
+        super.connect(newAddr, timeout);
+      }
+    };
+  }
+
+  /* @inheritDoc */
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (!(obj instanceof DummySocketFactory))
+      return false;
+    return true;
+  }
+
+  /* @inheritDoc */
+  @Override
+  public int hashCode() {
+    // Dummy hash code (to make find bugs happy)
+    return 53;
+  }
+}

src/test/hdfs-with-mr/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java

@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.authorize;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HDFSPolicyProvider;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.TestMiniMRWithDFS;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
+
+import junit.framework.TestCase;
+
+public class TestServiceLevelAuthorization extends TestCase {
+  public void testServiceLevelAuthorization() throws Exception {
+    MiniDFSCluster dfs = null;
+    MiniMRCluster mr = null;
+    FileSystem fileSys = null;
+    try {
+      final int slaves = 4;
+
+      // Turn on service-level authorization
+      Configuration conf = new Configuration();
+      conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, 
+                    HadoopPolicyProvider.class, PolicyProvider.class);
+      conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, 
+                      true);
+      
+      // Start the mini clusters
+      dfs = new MiniDFSCluster(conf, slaves, true, null);
+      fileSys = dfs.getFileSystem();
+      JobConf mrConf = new JobConf(conf);
+      mr = new MiniMRCluster(slaves, fileSys.getUri().toString(), 1, 
+                             null, null, mrConf);
+
+      // Run examples
+      TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf));
+      TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf));
+    } finally {
+      if (dfs != null) { dfs.shutdown(); }
+      if (mr != null) { mr.shutdown();
+      }
+    }
+  }
+  
+  private static final String DUMMY_ACL = "nouser nogroup";
+  private static final String UNKNOWN_USER = "dev,null";
+  
+  private void rewriteHadoopPolicyFile(File policyFile) throws IOException {
+    FileWriter fos = new FileWriter(policyFile);
+    PolicyProvider policyProvider = new HDFSPolicyProvider();
+    fos.write("<configuration>\n");
+    for (Service service : policyProvider.getServices()) {
+      String key = service.getServiceKey();
+      String value ="*";
+      if (key.equals("security.refresh.policy.protocol.acl")) {
+        value = DUMMY_ACL;
+      }
+      fos.write("<property><name>"+ key + "</name><value>" + value + 
+                "</value></property>\n");
+      System.err.println("<property><name>"+ key + "</name><value>" + value + 
+          "</value></property>\n");
+    }
+    fos.write("</configuration>\n");
+    fos.close();
+  }
+  
+  private void refreshPolicy(Configuration conf)  throws IOException {
+    DFSAdmin dfsAdmin = new DFSAdmin(conf);
+    dfsAdmin.refreshServiceAcl();
+  }
+  
+  public void testRefresh() throws Exception {
+    MiniDFSCluster dfs = null;
+    try {
+      final int slaves = 4;
+
+      // Turn on service-level authorization
+      Configuration conf = new Configuration();
+      conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, 
+                    HDFSPolicyProvider.class, PolicyProvider.class);
+      conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, 
+                      true);
+      
+      // Start the mini dfs cluster
+      dfs = new MiniDFSCluster(conf, slaves, true, null);
+
+      // Refresh the service level authorization policy
+      refreshPolicy(conf);
+      
+      // Simulate an 'edit' of hadoop-policy.xml
+      String confDir = System.getProperty("test.build.extraconf", 
+                                          "build/test/extraconf");
+      File policyFile = new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE);
+      String policyFileCopy = ConfiguredPolicy.HADOOP_POLICY_FILE + ".orig";
+      FileUtil.copy(policyFile, FileSystem.getLocal(conf),   // first save original 
+                    new Path(confDir, policyFileCopy), false, conf);
+      rewriteHadoopPolicyFile(                               // rewrite the file
+          new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
+      
+      // Refresh the service level authorization policy
+      refreshPolicy(conf);
+      
+      // Refresh the service level authorization policy once again, 
+      // this time it should fail!
+      try {
+        // Note: hadoop-policy.xml for tests has 
+        // security.refresh.policy.protocol.acl = ${user.name}
+        conf.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, UNKNOWN_USER);
+        refreshPolicy(conf);
+        fail("Refresh of NameNode's policy file cannot be successful!");
+      } catch (RemoteException re) {
+        System.out.println("Good, refresh worked... refresh failed with: " + 
+                           StringUtils.stringifyException(re.unwrapRemoteException()));
+      } finally {
+        // Reset to original hadoop-policy.xml
+        FileUtil.fullyDelete(new File(confDir, 
+            ConfiguredPolicy.HADOOP_POLICY_FILE));
+        FileUtil.replaceFile(new File(confDir, policyFileCopy), new File(confDir, ConfiguredPolicy.HADOOP_POLICY_FILE));
+      }
+    } finally {
+      if (dfs != null) { dfs.shutdown(); }
+    }
+  }
+
+}

src/test/hdfs-with-mr/org/apache/hadoop/test/AllTestDriver.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.util.ProgramDriver;
+
+
+@Deprecated
+//Class to be removed after the project split
+public class AllTestDriver {
+  
+  /**
+   * A description of the test program for running all the tests using jar file
+   */
+  public static void main(String argv[]){
+    ProgramDriver pd = new ProgramDriver();
+    new CoreTestDriver(pd);
+    new HdfsTestDriver(pd);
+    new HdfsWithMRTestDriver(pd);
+    new MapredTestDriver(pd);
+    
+    try {
+      pd.driver(argv);
+    } catch (Throwable e) {
+      e.printStackTrace();
+    }
+  }
+
+}
+

src/test/hdfs-with-mr/org/apache/hadoop/test/HdfsWithMRTestDriver.java

@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.fs.DFSCIOTest;
+import org.apache.hadoop.fs.DistributedFSCheck;
+import org.apache.hadoop.fs.TestDFSIO;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.hdfs.NNBench;
+import org.apache.hadoop.io.FileBench;
+import org.apache.hadoop.util.ProgramDriver;
+
+/*
+ * Driver for HDFS tests, which require map-reduce to run.
+ */
+public class HdfsWithMRTestDriver {
+  
+  
+  private ProgramDriver pgd;
+
+  public HdfsWithMRTestDriver() {
+    this(new ProgramDriver());
+  }
+  
+  public HdfsWithMRTestDriver(ProgramDriver pgd) {
+    this.pgd = pgd;
+    try {
+      pgd.addClass("nnbench", NNBench.class, 
+          "A benchmark that stresses the namenode.");
+      pgd.addClass("testfilesystem", TestFileSystem.class, 
+          "A test for FileSystem read/write.");
+      pgd.addClass("TestDFSIO", TestDFSIO.class, 
+          "Distributed i/o benchmark.");
+      pgd.addClass("DFSCIOTest", DFSCIOTest.class, "" +
+          "Distributed i/o benchmark of libhdfs.");
+      pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, 
+          "Distributed checkup of the file system consistency.");
+      pgd.addClass("filebench", FileBench.class, 
+          "Benchmark SequenceFile(Input|Output)Format " +
+          "(block,record compressed and uncompressed), " +
+          "Text(Input|Output)Format (compressed and uncompressed)");
+    } catch(Throwable e) {
+      e.printStackTrace();
+    }
+  }
+
+  public void run(String argv[]) {
+    try {
+      pgd.driver(argv);
+    } catch(Throwable e) {
+      e.printStackTrace();
+    }
+  }
+
+  public static void main(String argv[]){
+    new HdfsWithMRTestDriver().run(argv);
+  }
+}
+

src/test/hdfs-with-mr/org/apache/hadoop/tools/TestDistCh.java

@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.TaskTracker;
+import org.apache.log4j.Level;
+
+public class TestDistCh extends junit.framework.TestCase {
+  {
+    ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
+        ).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.OFF);
+    ((Log4JLogger)TaskTracker.LOG).getLogger().setLevel(Level.OFF);
+  }
+
+  static final Long RANDOM_NUMBER_GENERATOR_SEED = null;
+
+  private static final Random RANDOM = new Random();
+  static {
+    final long seed = RANDOM_NUMBER_GENERATOR_SEED == null?
+        RANDOM.nextLong(): RANDOM_NUMBER_GENERATOR_SEED;
+    System.out.println("seed=" + seed);
+    RANDOM.setSeed(seed);
+  }
+
+  static final String TEST_ROOT_DIR =
+    new Path(System.getProperty("test.build.data","/tmp")
+        ).toString().replace(' ', '+');
+
+  static final int NUN_SUBS = 5;
+
+  static class FileTree {
+    private final FileSystem fs;
+    private final String root;
+    private final Path rootdir;
+    private int fcount = 0;
+
+    Path createSmallFile(Path dir) throws IOException {
+      final Path f = new Path(dir, "f" + ++fcount);
+      assertTrue(!fs.exists(f));
+      final DataOutputStream out = fs.create(f);
+      try {
+        out.writeBytes("createSmallFile: f=" + f);
+      } finally {
+        out.close();
+      }
+      assertTrue(fs.exists(f));
+      return f;
+    }
+
+    Path mkdir(Path dir) throws IOException {
+      assertTrue(fs.mkdirs(dir));
+      assertTrue(fs.getFileStatus(dir).isDir());
+      return dir;
+    }
+    
+    FileTree(FileSystem fs, String name) throws IOException {
+      this.fs = fs;
+      this.root = "/test/" + name;
+      this.rootdir = mkdir(new Path(root));
+  
+      for(int i = 0; i < 3; i++) {
+        createSmallFile(rootdir);
+      }
+      
+      for(int i = 0; i < NUN_SUBS; i++) {
+        final Path sub = mkdir(new Path(root, "sub" + i));
+        int num_files = RANDOM.nextInt(3);
+        for(int j = 0; j < num_files; j++) {
+          createSmallFile(sub);
+        }
+      }
+      
+      System.out.println("rootdir = " + rootdir);
+    }
+  }
+
+  static class ChPermissionStatus extends PermissionStatus {
+    ChPermissionStatus(FileStatus filestatus) {
+      this(filestatus, "", "", "");
+    }
+
+    ChPermissionStatus(FileStatus filestatus, String owner, String group, String permission) {
+      super("".equals(owner)? filestatus.getOwner(): owner, 
+          "".equals(group)? filestatus.getGroup(): group,
+          "".equals(permission)? filestatus.getPermission(): new FsPermission(Short.parseShort(permission, 8)));
+    }
+  }
+  
+  public void testDistCh() throws Exception {
+    final Configuration conf = new Configuration();
+    final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+    final FileSystem fs = cluster.getFileSystem();
+    final MiniMRCluster mr = new MiniMRCluster(2, fs.getUri().toString(), 1);
+    final FsShell shell = new FsShell(conf);
+    
+    try {
+      final FileTree tree = new FileTree(fs, "testDistCh");
+      final FileStatus rootstatus = fs.getFileStatus(tree.rootdir);
+
+      runLsr(shell, tree.root, 0);
+
+      //generate random arguments
+      final String[] args = new String[RANDOM.nextInt(NUN_SUBS-1) + 1];
+      final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
+      final List<Integer> indices = new LinkedList<Integer>();
+      for(int i = 0; i < NUN_SUBS; i++) {
+        indices.add(i);
+      }
+      for(int i = 0; i < args.length; i++) {
+        final int index = indices.remove(RANDOM.nextInt(indices.size()));
+        final String sub = "sub" + index;
+        final boolean changeOwner = RANDOM.nextBoolean();
+        final boolean changeGroup = RANDOM.nextBoolean();
+        final boolean changeMode = !changeOwner && !changeGroup? true: RANDOM.nextBoolean();
+        
+        final String owner = changeOwner? sub: "";
+        final String group = changeGroup? sub: "";
+        final String permission = changeMode? RANDOM.nextInt(8) + "" + RANDOM.nextInt(8) + "" + RANDOM.nextInt(8): "";
+
+        args[i] = tree.root + "/" + sub + ":" + owner + ":" + group + ":" + permission;
+        newstatus[index] = new ChPermissionStatus(rootstatus, owner, group, permission);
+      }
+      for(int i = 0; i < NUN_SUBS; i++) {
+        if (newstatus[i] == null) {
+          newstatus[i] = new ChPermissionStatus(rootstatus);
+        }
+      }
+      System.out.println("args=" + Arrays.asList(args).toString().replace(",", ",\n  "));
+      System.out.println("newstatus=" + Arrays.asList(newstatus).toString().replace(",", ",\n  "));
+
+      //run DistCh
+      new DistCh(mr.createJobConf()).run(args);
+      runLsr(shell, tree.root, 0);
+
+      //check results
+      for(int i = 0; i < NUN_SUBS; i++) {
+        Path sub = new Path(tree.root + "/sub" + i);
+        checkFileStatus(newstatus[i], fs.getFileStatus(sub));
+        for(FileStatus status : fs.listStatus(sub)) {
+          checkFileStatus(newstatus[i], status);
+        }
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+
+  static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
+    assertEquals(expected.getUserName(), actual.getOwner());
+    assertEquals(expected.getGroupName(), actual.getGroup());
+    FsPermission perm = expected.getPermission(); 
+    if (!actual.isDir()) {
+      perm = perm.applyUMask(UMASK);
+    }
+    assertEquals(perm, actual.getPermission());
+  }
+
+  private static String runLsr(final FsShell shell, String root, int returnvalue
+      ) throws Exception {
+    System.out.println("root=" + root + ", returnvalue=" + returnvalue);
+    final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
+    final PrintStream out = new PrintStream(bytes);
+    final PrintStream oldOut = System.out;
+    final PrintStream oldErr = System.err;
+    System.setOut(out);
+    System.setErr(out);
+    final String results;
+    try {
+      assertEquals(returnvalue, shell.run(new String[]{"-lsr", root}));
+      results = bytes.toString();
+    } finally {
+      IOUtils.closeStream(out);
+      System.setOut(oldOut);
+      System.setErr(oldErr);
+    }
+    System.out.println("results:\n" + results);
+    return results;
+  }
+}

src/webapps/datanode/browseBlock.jsp

@@ -0,0 +1,404 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="java.net.*"
+
+  import="org.apache.hadoop.hdfs.*"
+  import="org.apache.hadoop.hdfs.server.namenode.*"
+  import="org.apache.hadoop.hdfs.protocol.*"
+  import="org.apache.hadoop.security.AccessToken"
+  import="org.apache.hadoop.security.AccessTokenHandler"
+  import="org.apache.hadoop.util.*"
+%>
+
+<%!
+  static final DataNode datanode = DataNode.getDataNode();
+
+  public void generateFileDetails(JspWriter out, HttpServletRequest req) 
+    throws IOException {
+
+    long startOffset = 0;
+    int datanodePort;
+
+    final Long blockId = JspHelper.validateLong(req.getParameter("blockId"));
+    if (blockId == null) {
+      out.print("Invalid input (blockId absent)");
+      return;
+    }
+
+    String datanodePortStr = req.getParameter("datanodePort");
+    if (datanodePortStr == null) {
+      out.print("Invalid input (datanodePort absent)");
+      return;
+    }
+    datanodePort = Integer.parseInt(datanodePortStr);
+
+    String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
+    int namenodeInfoPort = -1;
+    if (namenodeInfoPortStr != null)
+      namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
+
+    final int chunkSizeToView = JspHelper.string2ChunkSizeToView(req.getParameter("chunkSizeToView"));
+
+    String startOffsetStr = req.getParameter("startOffset");
+    if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0)
+      startOffset = 0;
+    else startOffset = Long.parseLong(startOffsetStr);
+    
+    final String filename = JspHelper.validatePath(
+        req.getParameter("filename"));
+    if (filename == null) {
+      out.print("Invalid input");
+      return;
+    }
+
+    String blockSizeStr = req.getParameter("blockSize"); 
+    long blockSize = 0;
+    if (blockSizeStr == null || blockSizeStr.length() == 0) {
+      out.print("Invalid input");
+      return;
+    } 
+    blockSize = Long.parseLong(blockSizeStr);
+
+    final DFSClient dfs = new DFSClient(datanode.getNameNodeAddr(), JspHelper.conf);
+    List<LocatedBlock> blocks = 
+      dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
+    //Add the various links for looking at the file contents
+    //URL for downloading the full file
+    String downloadUrl = "http://" + req.getServerName() + ":" +
+                         + req.getServerPort() + "/streamFile?" + "filename=" +
+                         URLEncoder.encode(filename, "UTF-8");
+    out.print("<a name=\"viewOptions\"></a>");
+    out.print("<a href=\"" + downloadUrl + "\">Download this file</a><br>");
+    
+    DatanodeInfo chosenNode;
+    //URL for TAIL 
+    LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
+    try {
+      chosenNode = JspHelper.bestNode(lastBlk);
+    } catch (IOException e) {
+      out.print(e.toString());
+      dfs.close();
+      return;
+    }
+    String fqdn = 
+           InetAddress.getByName(chosenNode.getHost()).getCanonicalHostName();
+    String tailUrl = "http://" + fqdn + ":" +
+                     chosenNode.getInfoPort() + 
+                 "/tail.jsp?filename=" + URLEncoder.encode(filename, "UTF-8") +
+                 "&namenodeInfoPort=" + namenodeInfoPort +
+                 "&chunkSizeToView=" + chunkSizeToView +
+                 "&referrer=" + 
+          URLEncoder.encode(req.getRequestURL() + "?" + req.getQueryString(),
+                            "UTF-8");
+    out.print("<a href=\"" + tailUrl + "\">Tail this file</a><br>");
+
+    out.print("<form action=\"/browseBlock.jsp\" method=GET>");
+    out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
+    out.print("<input type=\"hidden\" name=\"blockId\" value=\"" + blockId +
+              "\">");
+    out.print("<input type=\"hidden\" name=\"blockSize\" value=\"" + 
+              blockSize + "\">");
+    out.print("<input type=\"hidden\" name=\"startOffset\" value=\"" + 
+              startOffset + "\">");
+    out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename +
+              "\">");
+    out.print("<input type=\"hidden\" name=\"datanodePort\" value=\"" + 
+              datanodePort+ "\">");
+    out.print("<input type=\"hidden\" name=\"namenodeInfoPort\" value=\"" +
+              namenodeInfoPort + "\">");
+    out.print("<input type=\"text\" name=\"chunkSizeToView\" value=" +
+              chunkSizeToView + " size=10 maxlength=10>");
+    out.print("&nbsp;&nbsp;<input type=\"submit\" name=\"submit\" value=\"Refresh\">");
+    out.print("</form>");
+    out.print("<hr>"); 
+    out.print("<a name=\"blockDetails\"></a>");
+    out.print("<B>Total number of blocks: "+blocks.size()+"</B><br>");
+    //generate a table and dump the info
+    out.println("\n<table>");
+    for (LocatedBlock cur : blocks) {
+      out.print("<tr>");
+      final String blockidstring = Long.toString(cur.getBlock().getBlockId());
+      blockSize = cur.getBlock().getNumBytes();
+      out.print("<td>"+blockidstring+":</td>");
+      DatanodeInfo[] locs = cur.getLocations();
+      for(int j=0; j<locs.length; j++) {
+        String datanodeAddr = locs[j].getName();
+        datanodePort = Integer.parseInt(datanodeAddr.substring(
+                                        datanodeAddr.indexOf(':') + 1, 
+                                    datanodeAddr.length())); 
+        fqdn = InetAddress.getByName(locs[j].getHost()).getCanonicalHostName();
+        String blockUrl = "http://"+ fqdn + ":" +
+                        locs[j].getInfoPort() +
+                        "/browseBlock.jsp?blockId=" + blockidstring +
+                        "&blockSize=" + blockSize +
+               "&filename=" + URLEncoder.encode(filename, "UTF-8")+ 
+                        "&datanodePort=" + datanodePort + 
+                        "&genstamp=" + cur.getBlock().getGenerationStamp() + 
+                        "&namenodeInfoPort=" + namenodeInfoPort +
+                        "&chunkSizeToView=" + chunkSizeToView;
+        out.print("<td>&nbsp</td>" 
+          + "<td><a href=\"" + blockUrl + "\">" + datanodeAddr + "</a></td>");
+      }
+      out.println("</tr>");
+    }
+    out.println("</table>");
+    out.print("<hr>");
+    String namenodeHost = datanode.getNameNodeAddr().getHostName();
+    out.print("<br><a href=\"http://" + 
+              InetAddress.getByName(namenodeHost).getCanonicalHostName() + ":" +
+              namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
+    dfs.close();
+  }
+
+  public void generateFileChunks(JspWriter out, HttpServletRequest req) 
+    throws IOException {
+    long startOffset = 0;
+    int datanodePort = 0; 
+
+    String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
+    int namenodeInfoPort = -1;
+    if (namenodeInfoPortStr != null)
+      namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
+
+    final String filename = JspHelper.validatePath(
+        req.getParameter("filename"));
+    if (filename == null) {
+      out.print("Invalid input (filename absent)");
+      return;
+    }
+    
+    final Long blockId = JspHelper.validateLong(req.getParameter("blockId"));
+    if (blockId == null) {
+      out.print("Invalid input (blockId absent)");
+      return;
+    }
+
+    final DFSClient dfs = new DFSClient(datanode.getNameNodeAddr(), JspHelper.conf);
+    
+    AccessToken accessToken = AccessToken.DUMMY_TOKEN;
+    if (JspHelper.conf
+        .getBoolean(AccessTokenHandler.STRING_ENABLE_ACCESS_TOKEN, false)) {
+      List<LocatedBlock> blks = dfs.namenode.getBlockLocations(filename, 0,
+          Long.MAX_VALUE).getLocatedBlocks();
+      if (blks == null || blks.size() == 0) {
+        out.print("Can't locate file blocks");
+        dfs.close();
+        return;
+      }
+      for (int i = 0; i < blks.size(); i++) {
+        if (blks.get(i).getBlock().getBlockId() == blockId) {
+          accessToken = blks.get(i).getAccessToken();
+          break;
+        }
+      }
+    }
+    
+    final Long genStamp = JspHelper.validateLong(req.getParameter("genstamp"));
+    if (genStamp == null) {
+      out.print("Invalid input (genstamp absent)");
+      return;
+    }
+
+    String blockSizeStr;
+    long blockSize = 0;
+    blockSizeStr = req.getParameter("blockSize"); 
+    if (blockSizeStr == null) {
+      out.print("Invalid input (blockSize absent)");
+      return;
+    }
+    blockSize = Long.parseLong(blockSizeStr);
+    
+    final int chunkSizeToView = JspHelper.string2ChunkSizeToView(req.getParameter("chunkSizeToView"));
+
+    String startOffsetStr = req.getParameter("startOffset");
+    if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0)
+      startOffset = 0;
+    else startOffset = Long.parseLong(startOffsetStr);
+
+    String datanodePortStr = req.getParameter("datanodePort");
+    if (datanodePortStr == null) {
+      out.print("Invalid input (datanodePort absent)");
+      return;
+    }
+    datanodePort = Integer.parseInt(datanodePortStr);
+    out.print("<h3>File: ");
+    JspHelper.printPathWithLinks(filename, out, namenodeInfoPort);
+    out.print("</h3><hr>");
+    String parent = new File(filename).getParent();
+    JspHelper.printGotoForm(out, namenodeInfoPort, parent);
+    out.print("<hr>");
+    out.print("<a href=\"http://" + req.getServerName() + ":" + 
+              req.getServerPort() + 
+              "/browseDirectory.jsp?dir=" + 
+              URLEncoder.encode(parent, "UTF-8") +
+              "&namenodeInfoPort=" + namenodeInfoPort + 
+              "\"><i>Go back to dir listing</i></a><br>");
+    out.print("<a href=\"#viewOptions\">Advanced view/download options</a><br>");
+    out.print("<hr>");
+
+    //Determine the prev & next blocks
+    long nextStartOffset = 0;
+    long nextBlockSize = 0;
+    String nextBlockIdStr = null;
+    String nextGenStamp = null;
+    String nextHost = req.getServerName();
+    int nextPort = req.getServerPort();
+    int nextDatanodePort = datanodePort;
+    //determine data for the next link
+    if (startOffset + chunkSizeToView >= blockSize) {
+      //we have to go to the next block from this point onwards
+      List<LocatedBlock> blocks = 
+        dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
+      for (int i = 0; i < blocks.size(); i++) {
+        if (blocks.get(i).getBlock().getBlockId() == blockId) {
+          if (i != blocks.size() - 1) {
+            LocatedBlock nextBlock = blocks.get(i+1);
+            nextBlockIdStr = Long.toString(nextBlock.getBlock().getBlockId());
+            nextGenStamp = Long.toString(nextBlock.getBlock().getGenerationStamp());
+            nextStartOffset = 0;
+            nextBlockSize = nextBlock.getBlock().getNumBytes();
+            DatanodeInfo d = JspHelper.bestNode(nextBlock);
+            String datanodeAddr = d.getName();
+            nextDatanodePort = Integer.parseInt(
+                                      datanodeAddr.substring(
+                                           datanodeAddr.indexOf(':') + 1, 
+                                      datanodeAddr.length())); 
+            nextHost = InetAddress.getByName(d.getHost()).getCanonicalHostName();
+            nextPort = d.getInfoPort(); 
+          }
+        }
+      }
+    } 
+    else {
+      //we are in the same block
+      nextBlockIdStr = blockId.toString();
+      nextStartOffset = startOffset + chunkSizeToView;
+      nextBlockSize = blockSize;
+      nextGenStamp = genStamp.toString();
+    }
+    String nextUrl = null;
+    if (nextBlockIdStr != null) {
+      nextUrl = "http://" + nextHost + ":" + 
+                nextPort + 
+                "/browseBlock.jsp?blockId=" + nextBlockIdStr +
+                "&blockSize=" + nextBlockSize + "&startOffset=" + 
+                nextStartOffset + 
+                "&genstamp=" + nextGenStamp +
+                "&filename=" + URLEncoder.encode(filename, "UTF-8") +
+                "&chunkSizeToView=" + chunkSizeToView + 
+                "&datanodePort=" + nextDatanodePort +
+                "&namenodeInfoPort=" + namenodeInfoPort;
+      out.print("<a href=\"" + nextUrl + "\">View Next chunk</a>&nbsp;&nbsp;");        
+    }
+    //determine data for the prev link
+    String prevBlockIdStr = null;
+    String prevGenStamp = null;
+    long prevStartOffset = 0;
+    long prevBlockSize = 0;
+    String prevHost = req.getServerName();
+    int prevPort = req.getServerPort();
+    int prevDatanodePort = datanodePort;
+    if (startOffset == 0) {
+      List<LocatedBlock> blocks = 
+        dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
+      for (int i = 0; i < blocks.size(); i++) {
+        if (blocks.get(i).getBlock().getBlockId() == blockId) {
+          if (i != 0) {
+            LocatedBlock prevBlock = blocks.get(i-1);
+            prevBlockIdStr = Long.toString(prevBlock.getBlock().getBlockId());
+            prevGenStamp = Long.toString(prevBlock.getBlock().getGenerationStamp());
+            prevStartOffset = prevBlock.getBlock().getNumBytes() - chunkSizeToView;
+            if (prevStartOffset < 0)
+              prevStartOffset = 0;
+            prevBlockSize = prevBlock.getBlock().getNumBytes();
+            DatanodeInfo d = JspHelper.bestNode(prevBlock);
+            String datanodeAddr = d.getName();
+            prevDatanodePort = Integer.parseInt(
+                                      datanodeAddr.substring(
+                                          datanodeAddr.indexOf(':') + 1, 
+                                      datanodeAddr.length())); 
+            prevHost = InetAddress.getByName(d.getHost()).getCanonicalHostName();
+            prevPort = d.getInfoPort();
+          }
+        }
+      }
+    }
+    else {
+      //we are in the same block
+      prevBlockIdStr = blockId.toString();
+      prevStartOffset = startOffset - chunkSizeToView;
+      if (prevStartOffset < 0) prevStartOffset = 0;
+      prevBlockSize = blockSize;
+      prevGenStamp = genStamp.toString();
+    }
+
+    String prevUrl = null;
+    if (prevBlockIdStr != null) {
+      prevUrl = "http://" + prevHost + ":" + 
+                prevPort + 
+                "/browseBlock.jsp?blockId=" + prevBlockIdStr + 
+                "&blockSize=" + prevBlockSize + "&startOffset=" + 
+                prevStartOffset + 
+                "&filename=" + URLEncoder.encode(filename, "UTF-8") + 
+                "&chunkSizeToView=" + chunkSizeToView +
+                "&genstamp=" + prevGenStamp +
+                "&datanodePort=" + prevDatanodePort +
+                "&namenodeInfoPort=" + namenodeInfoPort;
+      out.print("<a href=\"" + prevUrl + "\">View Prev chunk</a>&nbsp;&nbsp;");
+    }
+    out.print("<hr>");
+    out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
+    try {
+    JspHelper.streamBlockInAscii(
+            new InetSocketAddress(req.getServerName(), datanodePort), blockId, 
+            accessToken, genStamp, blockSize, startOffset, chunkSizeToView, out);
+    } catch (Exception e){
+        out.print(e);
+    }
+    out.print("</textarea>");
+    dfs.close();
+  }
+
+%>
+<html>
+<head>
+<%JspHelper.createTitle(out, request, request.getParameter("filename")); %>
+</head>
+<body onload="document.goto.dir.focus()">
+<% 
+   generateFileChunks(out,request);
+%>
+<hr>
+<% 
+   generateFileDetails(out,request);
+%>
+
+<h2>Local logs</h2>
+<a href="/logs/">Log</a> directory
+
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/datanode/browseDirectory.jsp

@@ -0,0 +1,192 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="java.net.*"
+
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.hdfs.*"
+  import="org.apache.hadoop.hdfs.server.namenode.*"
+  import="org.apache.hadoop.hdfs.protocol.*"
+  import="org.apache.hadoop.util.*"
+%>
+<%!
+  static final DataNode datanode = DataNode.getDataNode();
+  
+  public void generateDirectoryStructure( JspWriter out, 
+                                          HttpServletRequest req,
+                                          HttpServletResponse resp) 
+    throws IOException {
+    final String dir = JspHelper.validatePath(req.getParameter("dir"));
+    if (dir == null) {
+      out.print("Invalid input");
+      return;
+    }
+    
+    String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
+    int namenodeInfoPort = -1;
+    if (namenodeInfoPortStr != null)
+      namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
+    
+    final DFSClient dfs = new DFSClient(datanode.getNameNodeAddr(), JspHelper.conf);
+    String target = dir;
+    final FileStatus targetStatus = dfs.getFileInfo(target);
+    if (targetStatus == null) { // not exists
+      out.print("<h3>File or directory : " + target + " does not exist</h3>");
+      JspHelper.printGotoForm(out, namenodeInfoPort, target);
+    }
+    else {
+      if( !targetStatus.isDir() ) { // a file
+        List<LocatedBlock> blocks = 
+          dfs.namenode.getBlockLocations(dir, 0, 1).getLocatedBlocks();
+	      
+        LocatedBlock firstBlock = null;
+        DatanodeInfo [] locations = null;
+        if (blocks.size() > 0) {
+          firstBlock = blocks.get(0);
+          locations = firstBlock.getLocations();
+        }
+        if (locations == null || locations.length == 0) {
+          out.print("Empty file");
+        } else {
+          DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock);
+          String fqdn = InetAddress.getByName(chosenNode.getHost()).
+            getCanonicalHostName();
+          String datanodeAddr = chosenNode.getName();
+          int datanodePort = Integer.parseInt(
+                                              datanodeAddr.substring(
+                                                                     datanodeAddr.indexOf(':') + 1, 
+                                                                     datanodeAddr.length())); 
+          String redirectLocation = "http://"+fqdn+":" +
+            chosenNode.getInfoPort() + 
+            "/browseBlock.jsp?blockId=" +
+            firstBlock.getBlock().getBlockId() +
+            "&blockSize=" + firstBlock.getBlock().getNumBytes() +
+            "&genstamp=" + firstBlock.getBlock().getGenerationStamp() +
+            "&filename=" + URLEncoder.encode(dir, "UTF-8") + 
+            "&datanodePort=" + datanodePort + 
+            "&namenodeInfoPort=" + namenodeInfoPort;
+          resp.sendRedirect(redirectLocation);
+        }
+        return;
+      }
+      // directory
+      FileStatus[] files = dfs.listPaths(target);
+      //generate a table and dump the info
+      String [] headings = { "Name", "Type", "Size", "Replication", 
+                              "Block Size", "Modification Time",
+                              "Permission", "Owner", "Group" };
+      out.print("<h3>Contents of directory ");
+      JspHelper.printPathWithLinks(dir, out, namenodeInfoPort);
+      out.print("</h3><hr>");
+      JspHelper.printGotoForm(out, namenodeInfoPort, dir);
+      out.print("<hr>");
+	
+      File f = new File(dir);
+      String parent;
+      if ((parent = f.getParent()) != null)
+        out.print("<a href=\"" + req.getRequestURL() + "?dir=" + parent +
+                  "&namenodeInfoPort=" + namenodeInfoPort +
+                  "\">Go to parent directory</a><br>");
+	
+      if (files == null || files.length == 0) {
+        out.print("Empty directory");
+      }
+      else {
+        JspHelper.addTableHeader(out);
+        int row=0;
+        JspHelper.addTableRow(out, headings, row++);
+        String cols [] = new String[headings.length];
+        for (int i = 0; i < files.length; i++) {
+          //Get the location of the first block of the file
+          if (files[i].getPath().toString().endsWith(".crc")) continue;
+          if (!files[i].isDir()) {
+            cols[1] = "file";
+            cols[2] = StringUtils.byteDesc(files[i].getLen());
+            cols[3] = Short.toString(files[i].getReplication());
+            cols[4] = StringUtils.byteDesc(files[i].getBlockSize());
+          }
+          else {
+            cols[1] = "dir";
+            cols[2] = "";
+            cols[3] = "";
+            cols[4] = "";
+          }
+          String datanodeUrl = req.getRequestURL()+"?dir="+
+              URLEncoder.encode(files[i].getPath().toString(), "UTF-8") + 
+              "&namenodeInfoPort=" + namenodeInfoPort;
+          cols[0] = "<a href=\""+datanodeUrl+"\">"+files[i].getPath().getName()+"</a>";
+          cols[5] = FsShell.dateForm.format(new Date((files[i].getModificationTime())));
+          cols[6] = files[i].getPermission().toString();
+          cols[7] = files[i].getOwner();
+          cols[8] = files[i].getGroup();
+          JspHelper.addTableRow(out, cols, row++);
+        }
+        JspHelper.addTableFooter(out);
+      }
+    } 
+    String namenodeHost = datanode.getNameNodeAddr().getHostName();
+    out.print("<br><a href=\"http://" + 
+              InetAddress.getByName(namenodeHost).getCanonicalHostName() + ":" +
+              namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
+    dfs.close();
+  }
+
+%>
+
+<html>
+<head>
+<style type=text/css>
+<!--
+body 
+  {
+  font-face:sanserif;
+  }
+-->
+</style>
+<%JspHelper.createTitle(out, request, request.getParameter("dir")); %>
+</head>
+
+<body onload="document.goto.dir.focus()">
+<% 
+  try {
+    generateDirectoryStructure(out,request,response);
+  }
+  catch(IOException ioe) {
+    String msg = ioe.getLocalizedMessage();
+    int i = msg.indexOf("\n");
+    if (i >= 0) {
+      msg = msg.substring(0, i);
+    }
+    out.print("<h3>" + msg + "</h3>");
+  }
+%>
+<hr>
+
+<h2>Local logs</h2>
+<a href="/logs/">Log</a> directory
+
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/datanode/tail.jsp

@@ -0,0 +1,135 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="java.net.*"
+
+  import="org.apache.hadoop.hdfs.*"
+  import="org.apache.hadoop.hdfs.server.namenode.*"
+  import="org.apache.hadoop.hdfs.protocol.*"
+  import="org.apache.hadoop.security.AccessToken"
+  import="org.apache.hadoop.util.*"
+  import="org.apache.hadoop.net.NetUtils"
+%>
+
+<%!
+  static final DataNode datanode = DataNode.getDataNode();
+
+  public void generateFileChunks(JspWriter out, HttpServletRequest req) 
+    throws IOException {
+    final String referrer = JspHelper.validateURL(req.getParameter("referrer"));
+    boolean noLink = false;
+    if (referrer == null) {
+      noLink = true;
+    }
+
+    final String filename = JspHelper.validatePath(
+        req.getParameter("filename"));
+    if (filename == null) {
+      out.print("Invalid input (file name absent)");
+      return;
+    }
+
+    String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
+    int namenodeInfoPort = -1;
+    if (namenodeInfoPortStr != null)
+      namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
+    
+    final int chunkSizeToView = JspHelper.string2ChunkSizeToView(req.getParameter("chunkSizeToView"));
+
+    if (!noLink) {
+      out.print("<h3>Tail of File: ");
+      JspHelper.printPathWithLinks(filename, out, namenodeInfoPort);
+	    out.print("</h3><hr>");
+      out.print("<a href=\"" + referrer + "\">Go Back to File View</a><hr>");
+    }
+    else {
+      out.print("<h3>" + filename + "</h3>");
+    }
+    out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
+    out.print("<input type=\"text\" name=\"chunkSizeToView\" value=" +
+              chunkSizeToView + " size=10 maxlength=10>");
+    out.print("&nbsp;&nbsp;<input type=\"submit\" name=\"submit\" value=\"Refresh\"><hr>");
+    out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename +
+              "\">");
+    out.print("<input type=\"hidden\" name=\"namenodeInfoPort\" value=\"" + namenodeInfoPort +
+    "\">");
+    if (!noLink)
+      out.print("<input type=\"hidden\" name=\"referrer\" value=\"" + 
+                referrer+ "\">");
+
+    //fetch the block from the datanode that has the last block for this file
+    final DFSClient dfs = new DFSClient(datanode.getNameNodeAddr(), JspHelper.conf);
+    List<LocatedBlock> blocks = 
+      dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
+    if (blocks == null || blocks.size() == 0) {
+      out.print("No datanodes contain blocks of file "+filename);
+      dfs.close();
+      return;
+    }
+    LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
+    long blockSize = lastBlk.getBlock().getNumBytes();
+    long blockId = lastBlk.getBlock().getBlockId();
+    AccessToken accessToken = lastBlk.getAccessToken();
+    long genStamp = lastBlk.getBlock().getGenerationStamp();
+    DatanodeInfo chosenNode;
+    try {
+      chosenNode = JspHelper.bestNode(lastBlk);
+    } catch (IOException e) {
+      out.print(e.toString());
+      dfs.close();
+      return;
+    }      
+    InetSocketAddress addr = NetUtils.createSocketAddr(chosenNode.getName());
+    //view the last chunkSizeToView bytes while Tailing
+    final long startOffset = blockSize >= chunkSizeToView? blockSize - chunkSizeToView: 0;
+
+    out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
+    JspHelper.streamBlockInAscii(addr, blockId, accessToken, genStamp, blockSize, startOffset, chunkSizeToView, out);
+    out.print("</textarea>");
+    dfs.close();
+  }
+
+%>
+
+
+
+<html>
+<head>
+<%JspHelper.createTitle(out, request, request.getParameter("filename")); %>
+</head>
+<body>
+<form action="/tail.jsp" method="GET">
+<% 
+   generateFileChunks(out,request);
+%>
+</form>
+<hr>
+
+<h2>Local logs</h2>
+<a href="/logs/">Log</a> directory
+
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/hdfs/dfshealth.jsp

@@ -0,0 +1,280 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.hdfs.*"
+  import="org.apache.hadoop.hdfs.server.namenode.*"
+  import="org.apache.hadoop.hdfs.server.datanode.*"
+  import="org.apache.hadoop.hdfs.server.common.Storage"
+  import="org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory"
+  import="org.apache.hadoop.hdfs.protocol.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.DateFormat"
+  import="java.lang.Math"
+  import="java.net.URLEncoder"
+%>
+<%!
+  int rowNum = 0;
+  int colNum = 0;
+
+  String rowTxt() { colNum = 0;
+      return "<tr class=\"" + (((rowNum++)%2 == 0)? "rowNormal" : "rowAlt")
+          + "\"> "; }
+  String colTxt() { return "<td id=\"col" + ++colNum + "\"> "; }
+  void counterReset () { colNum = 0; rowNum = 0 ; }
+
+  long diskBytes = 1024 * 1024 * 1024;
+  String diskByteStr = "GB";
+
+  String sorterField = null;
+  String sorterOrder = null;
+
+  String NodeHeaderStr(String name) {
+      String ret = "class=header";
+      String order = "ASC";
+      if ( name.equals( sorterField ) ) {
+          ret += sorterOrder;
+          if ( sorterOrder.equals("ASC") )
+              order = "DSC";
+      }
+      ret += " onClick=\"window.document.location=" +
+          "'/dfshealth.jsp?sorter/field=" + name + "&sorter/order=" +
+          order + "'\" title=\"sort on this column\"";
+      
+      return ret;
+  }
+      
+  public void generateNodeData( JspWriter out, DatanodeDescriptor d,
+                                    String suffix, boolean alive,
+                                    int nnHttpPort )
+    throws IOException {
+      
+    /* Say the datanode is dn1.hadoop.apache.org with ip 192.168.0.5
+       we use:
+       1) d.getHostName():d.getPort() to display.
+           Domain and port are stripped if they are common across the nodes.
+           i.e. "dn1"
+       2) d.getHost():d.Port() for "title".
+          i.e. "192.168.0.5:50010"
+       3) d.getHostName():d.getInfoPort() for url.
+          i.e. "http://dn1.hadoop.apache.org:50075/..."
+          Note that "d.getHost():d.getPort()" is what DFS clients use
+          to interact with datanodes.
+    */
+    // from nn_browsedfscontent.jsp:
+    String url = "http://" + d.getHostName() + ":" + d.getInfoPort() +
+                 "/browseDirectory.jsp?namenodeInfoPort=" +
+                 nnHttpPort + "&dir=" +
+                 URLEncoder.encode("/", "UTF-8");
+     
+    String name = d.getHostName() + ":" + d.getPort();
+    if ( !name.matches( "\\d+\\.\\d+.\\d+\\.\\d+.*" ) ) 
+        name = name.replaceAll( "\\.[^.:]*", "" );    
+    int idx = (suffix != null && name.endsWith( suffix )) ?
+        name.indexOf( suffix ) : -1;
+    
+    out.print( rowTxt() + "<td class=\"name\"><a title=\""
+               + d.getHost() + ":" + d.getPort() +
+               "\" href=\"" + url + "\">" +
+               (( idx > 0 ) ? name.substring(0, idx) : name) + "</a>" +
+               (( alive ) ? "" : "\n") );
+    if ( !alive )
+        return;
+    
+    long c = d.getCapacity();
+    long u = d.getDfsUsed();
+    long nu = d.getNonDfsUsed();
+    long r = d.getRemaining();
+    String percentUsed = StringUtils.limitDecimalTo2(d.getDfsUsedPercent());    
+    String percentRemaining = StringUtils.limitDecimalTo2(d.getRemainingPercent());    
+    
+    String adminState = (d.isDecommissioned() ? "Decommissioned" :
+                         (d.isDecommissionInProgress() ? "Decommission In Progress":
+                          "In Service"));
+    
+    long timestamp = d.getLastUpdate();
+    long currentTime = System.currentTimeMillis();
+    out.print("<td class=\"lastcontact\"> " +
+              ((currentTime - timestamp)/1000) +
+              "<td class=\"adminstate\">" +
+              adminState +
+              "<td align=\"right\" class=\"capacity\">" +
+              StringUtils.limitDecimalTo2(c*1.0/diskBytes) +
+              "<td align=\"right\" class=\"used\">" +
+              StringUtils.limitDecimalTo2(u*1.0/diskBytes) +      
+              "<td align=\"right\" class=\"nondfsused\">" +
+              StringUtils.limitDecimalTo2(nu*1.0/diskBytes) +      
+              "<td align=\"right\" class=\"remaining\">" +
+              StringUtils.limitDecimalTo2(r*1.0/diskBytes) +      
+              "<td align=\"right\" class=\"pcused\">" + percentUsed +
+              "<td class=\"pcused\">" +
+              ServletUtil.percentageGraph( (int)Double.parseDouble(percentUsed) , 100) +
+              "<td align=\"right\" class=\"pcremaining`\">" + percentRemaining +
+              "<td title=" + "\"blocks scheduled : " + d.getBlocksScheduled() + 
+              "\" class=\"blocks\">" + d.numBlocks() + "\n");
+  }
+  
+  
+  public void generateConfReport( JspWriter out,
+		  NameNode nn,
+		  HttpServletRequest request)
+  throws IOException {
+	  FSNamesystem fsn = nn.getNamesystem();
+	  long underReplicatedBlocks = fsn.getUnderReplicatedBlocks();
+	  FSImage fsImage = fsn.getFSImage();
+	  List<Storage.StorageDirectory> removedStorageDirs = fsImage.getRemovedStorageDirs();
+	  String storageDirsSizeStr="", removedStorageDirsSizeStr="", storageDirsStr="", removedStorageDirsStr="", storageDirsDiv="", removedStorageDirsDiv="";
+
+	  //FS Image storage configuration
+	  out.print("<h3> " + nn.getRole() + " Storage: </h3>");
+	  out.print("<div id=\"dfstable\"> <table border=1 cellpadding=10 cellspacing=0 title=\"NameNode Storage\">\n"+
+	  "<thead><tr><td><b>Storage Directory</b></td><td><b>Type</b></td><td><b>State</b></td></tr></thead>");
+	  
+	  StorageDirectory st =null;
+	  for (Iterator<StorageDirectory> it = fsImage.dirIterator(); it.hasNext();) {
+	      st = it.next();
+	      String dir = "" +  st.getRoot();
+		  String type = "" + st.getStorageDirType();
+		  out.print("<tr><td>"+dir+"</td><td>"+type+"</td><td>Active</td></tr>");
+	  }
+	  
+	  long storageDirsSize = removedStorageDirs.size();
+	  for(int i=0; i< storageDirsSize; i++){
+		  st = removedStorageDirs.get(i);
+		  String dir = "" +  st.getRoot();
+		  String type = "" + st.getStorageDirType();
+		  out.print("<tr><td>"+dir+"</td><td>"+type+"</td><td><font color=red>Failed</font></td></tr>");
+	  }
+	  
+	  out.print("</table></div><br>\n");
+  }
+
+
+  public void generateDFSHealthReport(JspWriter out,
+                                      NameNode nn,
+                                      HttpServletRequest request)
+                                      throws IOException {
+    FSNamesystem fsn = nn.getNamesystem();
+    ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+    ArrayList<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
+    fsn.DFSNodesStatus(live, dead);
+
+    sorterField = request.getParameter("sorter/field");
+    sorterOrder = request.getParameter("sorter/order");
+    if ( sorterField == null )
+        sorterField = "name";
+    if ( sorterOrder == null )
+        sorterOrder = "ASC";
+
+    // Find out common suffix. Should this be before or after the sort?
+    String port_suffix = null;
+    if ( live.size() > 0 ) {
+        String name = live.get(0).getName();
+        int idx = name.indexOf(':');
+        if ( idx > 0 ) {
+            port_suffix = name.substring( idx );
+        }
+        
+        for ( int i=1; port_suffix != null && i < live.size(); i++ ) {
+            if ( live.get(i).getName().endsWith( port_suffix ) == false ) {
+                port_suffix = null;
+                break;
+            }
+        }
+    }
+        
+    counterReset();
+    long[] fsnStats = fsn.getStats(); 
+    long total = fsnStats[0];
+    long remaining = fsnStats[2];
+    long used = fsnStats[1];
+    long nonDFS = total - remaining - used;
+	nonDFS = nonDFS < 0 ? 0 : nonDFS; 
+    float percentUsed = total <= 0 
+        ? 0f : ((float)used * 100.0f)/(float)total;
+    float percentRemaining = total <= 0 
+        ? 100f : ((float)remaining * 100.0f)/(float)total;
+
+    out.print( "<div id=\"dfstable\"> <table>\n" +
+	       rowTxt() + colTxt() + "Configured Capacity" + colTxt() + ":" + colTxt() +
+	       StringUtils.byteDesc( total ) +
+	       rowTxt() + colTxt() + "DFS Used" + colTxt() + ":" + colTxt() +
+	       StringUtils.byteDesc( used ) +
+	       rowTxt() + colTxt() + "Non DFS Used" + colTxt() + ":" + colTxt() +
+	       StringUtils.byteDesc( nonDFS ) +
+	       rowTxt() + colTxt() + "DFS Remaining" + colTxt() + ":" + colTxt() +
+	       StringUtils.byteDesc( remaining ) +
+	       rowTxt() + colTxt() + "DFS Used%" + colTxt() + ":" + colTxt() +
+	       StringUtils.limitDecimalTo2(percentUsed) + " %" +
+	       rowTxt() + colTxt() + "DFS Remaining%" + colTxt() + ":" + colTxt() +
+	       StringUtils.limitDecimalTo2(percentRemaining) + " %" +
+	       rowTxt() + colTxt() +
+	       		"<a href=\"dfsnodelist.jsp?whatNodes=LIVE\">Live Nodes</a> " +
+	       		colTxt() + ":" + colTxt() + live.size() +
+	       rowTxt() + colTxt() +
+	       		"<a href=\"dfsnodelist.jsp?whatNodes=DEAD\">Dead Nodes</a> " +
+	       		colTxt() + ":" + colTxt() + dead.size() +
+               "</table></div><br>\n" );
+    
+    if (live.isEmpty() && dead.isEmpty()) {
+        out.print("There are no datanodes in the cluster");
+    }
+  }%>
+
+<%
+  NameNode nn = (NameNode)application.getAttribute("name.node");
+  FSNamesystem fsn = nn.getNamesystem();
+  String namenodeRole = nn.getRole().toString();
+  String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+%>
+
+<html>
+
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+<title>Hadoop <%=namenodeRole%> <%=namenodeLabel%></title>
+    
+<body>
+<h1><%=namenodeRole%> '<%=namenodeLabel%>'</h1>
+<%= JspHelper.getVersionTable(fsn) %>
+<br />
+<b><a href="/nn_browsedfscontent.jsp">Browse the filesystem</a></b><br>
+<b><a href="/logs/"><%=namenodeRole%> Logs</a></b>
+
+<hr>
+<h3>Cluster Summary</h3>
+<b> <%= JspHelper.getSafeModeText(fsn)%> </b>
+<b> <%= JspHelper.getInodeLimitText(fsn)%> </b>
+<a class="warning"> <%= JspHelper.getWarningText(fsn)%></a>
+
+<%
+    generateDFSHealthReport(out, nn, request); 
+%>
+<hr>
+<%
+	generateConfReport(out, nn, request);
+%>
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/hdfs/dfsnodelist.jsp

@@ -0,0 +1,276 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+contentType="text/html; charset=UTF-8"
+	import="javax.servlet.*"
+	import="javax.servlet.http.*"
+	import="java.io.*"
+	import="java.util.*"
+	import="org.apache.hadoop.fs.*"
+	import="org.apache.hadoop.hdfs.*"
+	import="org.apache.hadoop.hdfs.server.common.*"
+	import="org.apache.hadoop.hdfs.server.namenode.*"
+	import="org.apache.hadoop.hdfs.server.datanode.*"
+	import="org.apache.hadoop.hdfs.protocol.*"
+	import="org.apache.hadoop.util.*"
+	import="java.text.DateFormat"
+	import="java.lang.Math"
+	import="java.net.URLEncoder"
+%>
+<%!
+	int rowNum = 0;
+	int colNum = 0;
+
+	String rowTxt() { colNum = 0;
+	return "<tr class=\"" + (((rowNum++)%2 == 0)? "rowNormal" : "rowAlt")
+	+ "\"> "; }
+	String colTxt() { return "<td id=\"col" + ++colNum + "\"> "; }
+	void counterReset () { colNum = 0; rowNum = 0 ; }
+
+	long diskBytes = 1024 * 1024 * 1024;
+	String diskByteStr = "GB";
+
+	String sorterField = null;
+	String sorterOrder = null;
+	String whatNodes = "LIVE";
+
+String NodeHeaderStr(String name) {
+	String ret = "class=header";
+	String order = "ASC";
+	if ( name.equals( sorterField ) ) {
+		ret += sorterOrder;
+		if ( sorterOrder.equals("ASC") )
+			order = "DSC";
+	}
+	ret += " onClick=\"window.document.location=" +
+	"'/dfsnodelist.jsp?whatNodes="+whatNodes+"&sorter/field=" + name + "&sorter/order=" +
+	order + "'\" title=\"sort on this column\"";
+
+	return ret;
+}
+
+public void generateNodeData( JspWriter out, DatanodeDescriptor d,
+		String suffix, boolean alive,
+		int nnHttpPort )
+throws IOException {
+
+	/* Say the datanode is dn1.hadoop.apache.org with ip 192.168.0.5
+we use:
+1) d.getHostName():d.getPort() to display.
+Domain and port are stripped if they are common across the nodes.
+i.e. "dn1"
+2) d.getHost():d.Port() for "title".
+i.e. "192.168.0.5:50010"
+3) d.getHostName():d.getInfoPort() for url.
+i.e. "http://dn1.hadoop.apache.org:50075/..."
+Note that "d.getHost():d.getPort()" is what DFS clients use
+to interact with datanodes.
+	 */
+	// from nn_browsedfscontent.jsp:
+	String url = "http://" + d.getHostName() + ":" + d.getInfoPort() +
+	"/browseDirectory.jsp?namenodeInfoPort=" +
+	nnHttpPort + "&dir=" +
+	URLEncoder.encode("/", "UTF-8");
+
+	String name = d.getHostName() + ":" + d.getPort();
+	if ( !name.matches( "\\d+\\.\\d+.\\d+\\.\\d+.*" ) ) 
+		name = name.replaceAll( "\\.[^.:]*", "" );    
+	int idx = (suffix != null && name.endsWith( suffix )) ?
+			name.indexOf( suffix ) : -1;
+
+			out.print( rowTxt() + "<td class=\"name\"><a title=\""
+					+ d.getHost() + ":" + d.getPort() +
+					"\" href=\"" + url + "\">" +
+					(( idx > 0 ) ? name.substring(0, idx) : name) + "</a>" +
+					(( alive ) ? "" : "\n") );
+			if ( !alive )
+				return;
+
+			long c = d.getCapacity();
+			long u = d.getDfsUsed();
+			long nu = d.getNonDfsUsed();
+			long r = d.getRemaining();
+			String percentUsed = StringUtils.limitDecimalTo2(d.getDfsUsedPercent());    
+			String percentRemaining = StringUtils.limitDecimalTo2(d.getRemainingPercent());    
+
+			String adminState = (d.isDecommissioned() ? "Decommissioned" :
+				(d.isDecommissionInProgress() ? "Decommission In Progress":
+				"In Service"));
+
+			long timestamp = d.getLastUpdate();
+			long currentTime = System.currentTimeMillis();
+			out.print("<td class=\"lastcontact\"> " +
+					((currentTime - timestamp)/1000) +
+					"<td class=\"adminstate\">" +
+					adminState +
+					"<td align=\"right\" class=\"capacity\">" +
+					StringUtils.limitDecimalTo2(c*1.0/diskBytes) +
+					"<td align=\"right\" class=\"used\">" +
+					StringUtils.limitDecimalTo2(u*1.0/diskBytes) +      
+					"<td align=\"right\" class=\"nondfsused\">" +
+					StringUtils.limitDecimalTo2(nu*1.0/diskBytes) +      
+					"<td align=\"right\" class=\"remaining\">" +
+					StringUtils.limitDecimalTo2(r*1.0/diskBytes) +      
+					"<td align=\"right\" class=\"pcused\">" + percentUsed +
+					"<td class=\"pcused\">" +
+					ServletUtil.percentageGraph( (int)Double.parseDouble(percentUsed) , 100) +
+					"<td align=\"right\" class=\"pcremaining`\">" + percentRemaining +
+					"<td title=" + "\"blocks scheduled : " + d.getBlocksScheduled() + 
+					"\" class=\"blocks\">" + d.numBlocks() + "\n");
+}
+
+
+
+public void generateDFSNodesList(JspWriter out, 
+		NameNode nn,
+		HttpServletRequest request)
+throws IOException {
+	ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();    
+	ArrayList<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
+	nn.getNamesystem().DFSNodesStatus(live, dead);
+
+	whatNodes = request.getParameter("whatNodes"); // show only live or only dead nodes
+	sorterField = request.getParameter("sorter/field");
+	sorterOrder = request.getParameter("sorter/order");
+	if ( sorterField == null )
+		sorterField = "name";
+	if ( sorterOrder == null )
+		sorterOrder = "ASC";
+
+	JspHelper.sortNodeList(live, sorterField, sorterOrder);
+	JspHelper.sortNodeList(dead, "name", "ASC");
+
+	// Find out common suffix. Should this be before or after the sort?
+	String port_suffix = null;
+	if ( live.size() > 0 ) {
+		String name = live.get(0).getName();
+		int idx = name.indexOf(':');
+		if ( idx > 0 ) {
+			port_suffix = name.substring( idx );
+		}
+
+		for ( int i=1; port_suffix != null && i < live.size(); i++ ) {
+			if ( live.get(i).getName().endsWith( port_suffix ) == false ) {
+				port_suffix = null;
+				break;
+			}
+		}
+	}
+
+	counterReset();
+
+	try {
+		Thread.sleep(1000);
+	} catch (InterruptedException e) {}
+
+	if (live.isEmpty() && dead.isEmpty()) {
+		out.print("There are no datanodes in the cluster");
+	}
+	else {
+
+		int nnHttpPort = nn.getHttpAddress().getPort();
+		out.print( "<div id=\"dfsnodetable\"> ");
+		if(whatNodes.equals("LIVE")) {
+
+			out.print( 
+					"<a name=\"LiveNodes\" id=\"title\">" +
+					"Live Datanodes : " + live.size() + "</a>" +
+			"<br><br>\n<table border=1 cellspacing=0>\n" );
+
+			counterReset();
+
+			if ( live.size() > 0 ) {
+
+				if ( live.get(0).getCapacity() > 1024 * diskBytes ) {
+					diskBytes *= 1024;
+					diskByteStr = "TB";
+				}
+
+				out.print( "<tr class=\"headerRow\"> <th " +
+						NodeHeaderStr("name") + "> Node <th " +
+						NodeHeaderStr("lastcontact") + "> Last <br>Contact <th " +
+						NodeHeaderStr("adminstate") + "> Admin State <th " +
+						NodeHeaderStr("capacity") + "> Configured <br>Capacity (" + 
+						diskByteStr + ") <th " + 
+						NodeHeaderStr("used") + "> Used <br>(" + 
+						diskByteStr + ") <th " + 
+						NodeHeaderStr("nondfsused") + "> Non DFS <br>Used (" + 
+						diskByteStr + ") <th " + 
+						NodeHeaderStr("remaining") + "> Remaining <br>(" + 
+						diskByteStr + ") <th " + 
+						NodeHeaderStr("pcused") + "> Used <br>(%) <th " + 
+						NodeHeaderStr("pcused") + "> Used <br>(%) <th " +
+						NodeHeaderStr("pcremaining") + "> Remaining <br>(%) <th " +
+						NodeHeaderStr("blocks") + "> Blocks\n" );
+
+				JspHelper.sortNodeList(live, sorterField, sorterOrder);
+				for ( int i=0; i < live.size(); i++ ) {
+					generateNodeData(out, live.get(i), port_suffix, true, nnHttpPort);
+				}
+			}
+			out.print("</table>\n");
+		} else {
+
+			out.print("<br> <a name=\"DeadNodes\" id=\"title\"> " +
+					" Dead Datanodes : " +dead.size() + "</a><br><br>\n");
+
+			if ( dead.size() > 0 ) {
+				out.print( "<table border=1 cellspacing=0> <tr id=\"row1\"> " +
+				"<td> Node \n" );
+
+				JspHelper.sortNodeList(dead, "name", "ASC");
+				for ( int i=0; i < dead.size() ; i++ ) {
+					generateNodeData(out, dead.get(i), port_suffix, false, nnHttpPort);
+				}
+
+				out.print("</table>\n");
+			}
+		}
+		out.print("</div>");
+	}
+}%>
+
+<%
+NameNode nn = (NameNode)application.getAttribute("name.node");
+String namenodeRole = nn.getRole().toString();
+FSNamesystem fsn = nn.getNamesystem();
+String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+%>
+
+<html>
+
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+<title>Hadoop <%=namenodeRole%> <%=namenodeLabel%></title>
+  
+<body>
+<h1><%=namenodeRole%> '<%=namenodeLabel%>'</h1>
+<%= JspHelper.getVersionTable(fsn) %>
+<br />
+<b><a href="/nn_browsedfscontent.jsp">Browse the filesystem</a></b><br>
+<b><a href="/logs/"><%=namenodeRole%> Logs</a></b><br>
+<b><a href=/dfshealth.jsp> Go back to DFS home</a></b>
+<hr>
+<%
+	generateDFSNodesList(out, nn, request); 
+%>
+
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/hdfs/index.html

@@ -0,0 +1,35 @@
+<meta HTTP-EQUIV="REFRESH" content="0;url=dfshealth.jsp"/>
+<html>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<head>
+<title>Hadoop Administration</title>
+</head>
+
+<body>
+
+<h1>Hadoop Administration</h1>
+
+<ul>
+
+<li><a href="dfshealth.jsp">DFS Health/Status</a></li>
+
+</ul>
+
+</body>
+
+</html>

src/webapps/hdfs/nn_browsedfscontent.jsp

@@ -0,0 +1,77 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.*"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.hdfs.*"
+  import="org.apache.hadoop.hdfs.server.namenode.*"
+  import="org.apache.hadoop.hdfs.server.datanode.*"
+  import="org.apache.hadoop.hdfs.protocol.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.DateFormat"
+  import="java.net.InetAddress"
+  import="java.net.URLEncoder"
+%>
+<%!
+  public void redirectToRandomDataNode(
+                            NameNode nn, 
+                            HttpServletResponse resp) throws IOException {
+    FSNamesystem fsn = nn.getNamesystem();
+    String datanode = fsn.randomDataNode();
+    String redirectLocation;
+    String nodeToRedirect;
+    int redirectPort;
+    if (datanode != null) {
+      redirectPort = Integer.parseInt(datanode.substring(datanode.indexOf(':') + 1));
+      nodeToRedirect = datanode.substring(0, datanode.indexOf(':'));
+    }
+    else {
+      nodeToRedirect = nn.getHttpAddress().getHostName();
+      redirectPort = nn.getHttpAddress().getPort();
+    }
+    String fqdn = InetAddress.getByName(nodeToRedirect).getCanonicalHostName();
+    redirectLocation = "http://" + fqdn + ":" + redirectPort + 
+                       "/browseDirectory.jsp?namenodeInfoPort=" + 
+                       nn.getHttpAddress().getPort() +
+                       "&dir=" + URLEncoder.encode("/", "UTF-8");
+    resp.sendRedirect(redirectLocation);
+  }
+%>
+
+<html>
+
+<title></title>
+
+<body>
+<% 
+  NameNode nn = (NameNode)application.getAttribute("name.node");
+  redirectToRandomDataNode(nn, response); 
+%>
+<hr>
+
+<h2>Local logs</h2>
+<a href="/logs/">Log</a> directory
+
+<%
+out.println(ServletUtil.htmlFooter());
+%>

src/webapps/secondary/index.html

@@ -0,0 +1,29 @@
+<meta HTTP-EQUIV="REFRESH" content="0;url=status.jsp"/>
+<html>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<head><title>Hadoop Administration</title></head>
+
+<body>
+<h1>Hadoop Administration</h1>
+
+<ul> 
+  <li><a href="status.jsp">Status</a></li> 
+</ul>
+
+</body> 
+</html>

src/webapps/secondary/status.jsp

@@ -0,0 +1,39 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="org.apache.hadoop.util.*"
+%>
+
+<html>
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+<title>Hadoop SecondaryNameNode</title>
+    
+<body>
+<h1>SecondaryNameNode</h1>
+<%= JspHelper.getVersionTable() %>
+<hr />
+<pre>
+<%= application.getAttribute("secondary.name.node").toString() %>
+</pre>
+
+<br />
+<b><a href="/logs/">Logs</a></b>
+<%= ServletUtil.htmlFooter() %>