MAPREDUCE-6663. [NNBench] Refactor nnbench as a Tool implementation. Contributed by Brahma Reddy Battula.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java

@@ -25,7 +25,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
-import java.net.InetAddress;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Iterator;
@@ -33,6 +32,7 @@ import java.util.StringTokenizer;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile.Writer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileOutputFormat;
@@ -54,6 +55,8 @@ import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This program executes a specified operation that applies load to 
@@ -74,49 +77,48 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
  *       must be run before running the other operations.
  */
 
-public class NNBench {
+public class NNBench extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(
           "org.apache.hadoop.hdfs.NNBench");
   
-  protected static String CONTROL_DIR_NAME = "control";
-  protected static String OUTPUT_DIR_NAME = "output";
-  protected static String DATA_DIR_NAME = "data";
-  protected static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log";
-  protected static final String NNBENCH_VERSION = "NameNode Benchmark 0.4";
-  
-  public static String operation = "none";
-  public static long numberOfMaps = 1l; // default is 1
-  public static long numberOfReduces = 1l; // default is 1
-  public static long startTime = 
+  private static String CONTROL_DIR_NAME = "control";
+  private static String OUTPUT_DIR_NAME = "output";
+  private static String DATA_DIR_NAME = "data";
+  static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log";
+  private static final String NNBENCH_VERSION = "NameNode Benchmark 0.4";
+
+  private String operation = "none";
+  private long numberOfMaps = 1l; // default is 1
+  private long numberOfReduces = 1l; // default is 1
+  private long startTime =
           System.currentTimeMillis() + (120 * 1000); // default is 'now' + 2min
-  public static long blockSize = 1l; // default is 1
-  public static int bytesToWrite = 0; // default is 0
-  public static long bytesPerChecksum = 1l; // default is 1
-  public static long numberOfFiles = 1l; // default is 1
-  public static short replicationFactorPerFile = 1; // default is 1
-  public static String baseDir = "/benchmarks/NNBench";  // default
-  public static boolean readFileAfterOpen = false; // default is to not read
-  
+  private long blockSize = 1l; // default is 1
+  private int bytesToWrite = 0; // default is 0
+  private long bytesPerChecksum = 1l; // default is 1
+  private long numberOfFiles = 1l; // default is 1
+  private short replicationFactorPerFile = 1; // default is 1
+  private String baseDir = "/benchmarks/NNBench";  // default
+  private boolean readFileAfterOpen = false; // default is to not read
+  private boolean isHelpMessage = false;
   // Supported operations
   private static final String OP_CREATE_WRITE = "create_write";
   private static final String OP_OPEN_READ = "open_read";
   private static final String OP_RENAME = "rename";
   private static final String OP_DELETE = "delete";
+  private static final int MAX_OPERATION_EXCEPTIONS = 1000;
   
   // To display in the format that matches the NN and DN log format
   // Example: 2007-10-26 00:01:19,853
   static SimpleDateFormat sdf = 
           new SimpleDateFormat("yyyy-MM-dd' 'HH:mm:ss','S");
-
-  private static Configuration config = new Configuration();
   
   /**
    * Clean up the files before a test run
    * 
    * @throws IOException on error
    */
-  private static void cleanupBeforeTestrun() throws IOException {
-    FileSystem tempFS = FileSystem.get(config);
+  private void cleanupBeforeTestrun() throws IOException {
+    FileSystem tempFS = FileSystem.get(getConf());
     
     // Delete the data directory only if it is the create/write operation
     if (operation.equals(OP_CREATE_WRITE)) {
@@ -133,8 +135,7 @@ public class NNBench {
    * 
    * @throws IOException on error
    */
-  private static void createControlFiles() throws IOException {
-    FileSystem tempFS = FileSystem.get(config);
+  private void createControlFiles() throws IOException {
     LOG.info("Creating " + numberOfMaps + " control files");
 
     for (int i = 0; i < numberOfMaps; i++) {
@@ -144,8 +145,9 @@ public class NNBench {
 
       SequenceFile.Writer writer = null;
       try {
-        writer = SequenceFile.createWriter(tempFS, config, filePath, Text.class, 
-                LongWritable.class, CompressionType.NONE);
+        writer = SequenceFile.createWriter(getConf(), Writer.file(filePath),
+            Writer.keyClass(Text.class), Writer.valueClass(LongWritable.class),
+            Writer.compression(CompressionType.NONE));
         writer.append(new Text(strFileName), new LongWritable(i));
       } finally {
         if (writer != null) {
@@ -208,23 +210,23 @@ public class NNBench {
    *   line's arguments
    * @param length total number of arguments
    */
-  public static void checkArgs(final int index, final int length) {
+  private static void checkArgs(final int index, final int length) {
     if (index == length) {
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException("Not enough arguments");
     }
   }
   
   /**
    * Parse input arguments
+   *  @param args array of command line's parameters to be parsed
    *
-   * @param args array of command line's parameters to be parsed
    */
-  public static void parseInputs(final String[] args) {
+  private void parseInputs(final String[] args) {
     // If there are no command line arguments, exit
     if (args.length == 0) {
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException("Give valid inputs");
     }
     
     // Parse command line args
@@ -263,7 +265,7 @@ public class NNBench {
         readFileAfterOpen = Boolean.parseBoolean(args[++i]);
       } else if (args[i].equals("-help")) {
         displayUsage();
-        System.exit(-1);
+        isHelpMessage = true;
       }
     }
     
@@ -281,31 +283,30 @@ public class NNBench {
     LOG.info("     Read file after open: " + readFileAfterOpen);
     
     // Set user-defined parameters, so the map method can access the values
-    config.set("test.nnbench.operation", operation);
-    config.setLong("test.nnbench.maps", numberOfMaps);
-    config.setLong("test.nnbench.reduces", numberOfReduces);
-    config.setLong("test.nnbench.starttime", startTime);
-    config.setLong("test.nnbench.blocksize", blockSize);
-    config.setInt("test.nnbench.bytestowrite", bytesToWrite);
-    config.setLong("test.nnbench.bytesperchecksum", bytesPerChecksum);
-    config.setLong("test.nnbench.numberoffiles", numberOfFiles);
-    config.setInt("test.nnbench.replicationfactor", 
+    getConf().set("test.nnbench.operation", operation);
+    getConf().setLong("test.nnbench.maps", numberOfMaps);
+    getConf().setLong("test.nnbench.reduces", numberOfReduces);
+    getConf().setLong("test.nnbench.starttime", startTime);
+    getConf().setLong("test.nnbench.blocksize", blockSize);
+    getConf().setInt("test.nnbench.bytestowrite", bytesToWrite);
+    getConf().setLong("test.nnbench.bytesperchecksum", bytesPerChecksum);
+    getConf().setLong("test.nnbench.numberoffiles", numberOfFiles);
+    getConf().setInt("test.nnbench.replicationfactor",
             (int) replicationFactorPerFile);
-    config.set("test.nnbench.basedir", baseDir);
-    config.setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen);
+    getConf().set("test.nnbench.basedir", baseDir);
+    getConf().setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen);
 
-    config.set("test.nnbench.datadir.name", DATA_DIR_NAME);
-    config.set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME);
-    config.set("test.nnbench.controldir.name", CONTROL_DIR_NAME);
+    getConf().set("test.nnbench.datadir.name", DATA_DIR_NAME);
+    getConf().set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME);
+    getConf().set("test.nnbench.controldir.name", CONTROL_DIR_NAME);
   }
   
   /**
    * Analyze the results
-   * 
    * @throws IOException on error
    */
-  private static void analyzeResults() throws IOException {
-    final FileSystem fs = FileSystem.get(config);
+  private int analyzeResults() throws IOException {
+    final FileSystem fs = FileSystem.get(getConf());
     Path reduceDir = new Path(baseDir, OUTPUT_DIR_NAME);
 
     long totalTimeAL1 = 0l;
@@ -322,32 +323,31 @@ public class NNBench {
     for (FileStatus status : fss) {
 
       Path reduceFile = status.getPath();
-      DataInputStream in;
-      in = new DataInputStream(fs.open(reduceFile));
-
-      BufferedReader lines;
-      lines = new BufferedReader(new InputStreamReader(in));
-
-      String line;
-      while ((line = lines.readLine()) != null) {
-        StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;");
-        String attr = tokens.nextToken();
-        if (attr.endsWith(":totalTimeAL1")) {
-          totalTimeAL1 = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":totalTimeAL2")) {
-          totalTimeAL2 = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":totalTimeTPmS")) {
-          totalTimeTPmS = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":latemaps")) {
-          lateMaps = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":numOfExceptions")) {
-          numOfExceptions = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":successfulFileOps")) {
-          successfulFileOps = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":mapStartTimeTPmS")) {
-          mapStartTimeTPmS = Long.parseLong(tokens.nextToken());
-        } else if (attr.endsWith(":mapEndTimeTPmS")) {
-          mapEndTimeTPmS = Long.parseLong(tokens.nextToken());
+      try (DataInputStream in = new DataInputStream(fs.open(reduceFile));
+          BufferedReader lines =
+              new BufferedReader(new InputStreamReader(in))) {
+
+        String line;
+        while ((line = lines.readLine()) != null) {
+          StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;");
+          String attr = tokens.nextToken();
+          if (attr.endsWith(":totalTimeAL1")) {
+            totalTimeAL1 = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":totalTimeAL2")) {
+            totalTimeAL2 = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":totalTimeTPmS")) {
+            totalTimeTPmS = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":latemaps")) {
+            lateMaps = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":numOfExceptions")) {
+            numOfExceptions = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":successfulFileOps")) {
+            successfulFileOps = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":mapStartTimeTPmS")) {
+            mapStartTimeTPmS = Long.parseLong(tokens.nextToken());
+          } else if (attr.endsWith(":mapEndTimeTPmS")) {
+            mapEndTimeTPmS = Long.parseLong(tokens.nextToken());
+          }
         }
       }
     }
@@ -444,25 +444,29 @@ public class NNBench {
     "             RAW DATA: # of exceptions: " + numOfExceptions,
     "" };
 
-    PrintStream res = new PrintStream(new FileOutputStream(
-            new File(DEFAULT_RES_FILE_NAME), true));
-    
-    // Write to a file and also dump to log
-    for(int i = 0; i < resultLines.length; i++) {
-      LOG.info(resultLines[i]);
-      res.println(resultLines[i]);
+    try (PrintStream res = new PrintStream(
+        new FileOutputStream(new File(DEFAULT_RES_FILE_NAME), true))) {
+      // Write to a file and also dump to log
+      for (String resultLine : resultLines) {
+        LOG.info(resultLine);
+        res.println(resultLine);
+      }
+    }
+    if(numOfExceptions >= MAX_OPERATION_EXCEPTIONS){
+      return -1;
     }
+    return 0;
   }
-  
+
   /**
    * Run the test
    * 
    * @throws IOException on error
    */
-  public static void runTests() throws IOException {
-    config.setLong("io.bytes.per.checksum", bytesPerChecksum);
+  private void runTests() throws IOException {
+    getConf().setLong("io.bytes.per.checksum", bytesPerChecksum);
     
-    JobConf job = new JobConf(config, NNBench.class);
+    JobConf job = new JobConf(getConf(), NNBench.class);
 
     job.setJobName("NNBench-" + operation);
     FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME));
@@ -487,7 +491,7 @@ public class NNBench {
   /**
    * Validate the inputs
    */
-  public static void validateInputs() {
+  private void validateInputs() {
     // If it is not one of the four operations, then fail
     if (!operation.equals(OP_CREATE_WRITE) &&
             !operation.equals(OP_OPEN_READ) &&
@@ -495,7 +499,8 @@ public class NNBench {
             !operation.equals(OP_DELETE)) {
       System.err.println("Error: Unknown operation: " + operation);
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Unknown operation: " + operation);
     }
     
     // If number of maps is a negative number, then fail
@@ -503,57 +508,66 @@ public class NNBench {
     if (numberOfMaps < 0) {
       System.err.println("Error: Number of maps must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Number of maps must be a positive number");
     }
     
     // If number of reduces is a negative number or 0, then fail
     if (numberOfReduces <= 0) {
       System.err.println("Error: Number of reduces must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Number of reduces must be a positive number");
     }
 
     // If blocksize is a negative number or 0, then fail
     if (blockSize <= 0) {
       System.err.println("Error: Block size must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Block size must be a positive number");
     }
     
     // If bytes to write is a negative number, then fail
     if (bytesToWrite < 0) {
       System.err.println("Error: Bytes to write must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Bytes to write must be a positive number");
     }
     
     // If bytes per checksum is a negative number, then fail
     if (bytesPerChecksum < 0) {
       System.err.println("Error: Bytes per checksum must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Bytes per checksum must be a positive number");
     }
     
     // If number of files is a negative number, then fail
     if (numberOfFiles < 0) {
       System.err.println("Error: Number of files must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Number of files must be a positive number");
     }
     
     // If replication factor is a negative number, then fail
     if (replicationFactorPerFile < 0) {
       System.err.println("Error: Replication factor must be a positive number");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Replication factor must be a positive number");
     }
     
     // If block size is not a multiple of bytesperchecksum, fail
     if (blockSize % bytesPerChecksum != 0) {
-      System.err.println("Error: Block Size in bytes must be a multiple of " +
-              "bytes per checksum: ");
+      System.err.println("Error: Block Size in bytes must be a multiple of "
+          + "bytes per checksum: ");
       displayUsage();
-      System.exit(-1);
+      throw new HadoopIllegalArgumentException(
+          "Error: Block Size in bytes must be a multiple of "
+              + "bytes per checksum:");
     }
   }
   /**
@@ -562,13 +576,22 @@ public class NNBench {
   * @param args array of command line arguments
   * @throws IOException indicates a problem with test startup
   */
-  public static void main(String[] args) throws IOException {
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new NNBench(), args);
+    System.exit(res);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
     // Display the application version string
     displayVersion();
 
     // Parse the inputs
     parseInputs(args);
-    
+    if (isHelpMessage) {
+      return 0;
+    }
+
     // Validate inputs
     validateInputs();
     
@@ -582,7 +605,7 @@ public class NNBench {
     runTests();
     
     // Analyze results
-    analyzeResults();
+    return analyzeResults();
   }
 
   
@@ -592,7 +615,6 @@ public class NNBench {
   static class NNBenchMapper extends Configured
           implements Mapper<Text, LongWritable, Text, Text> {
     FileSystem filesystem = null;
-    private String hostName = null;
 
     long numberOfFiles = 1l;
     long blkSize = 1l;
@@ -602,7 +624,6 @@ public class NNBench {
     String dataDirName = null;
     String op = null;
     boolean readFile = false;
-    final int MAX_OPERATION_EXCEPTIONS = 1000;
     
     // Data to collect from the operation
     int numOfExceptions = 0;
@@ -628,12 +649,6 @@ public class NNBench {
       } catch(Exception e) {
         throw new RuntimeException("Cannot get file system.", e);
       }
-      
-      try {
-        hostName = InetAddress.getLocalHost().getHostName();
-      } catch(Exception e) {
-        throw new RuntimeException("Error getting hostname", e);
-      }
     }
     
     /**
@@ -678,7 +693,7 @@ public class NNBench {
             LongWritable value,
             OutputCollector<Text, Text> output,
             Reporter reporter) throws IOException {
-      Configuration conf = filesystem.getConf();
+      Configuration conf = getConf();
       
       numberOfFiles = conf.getLong("test.nnbench.numberoffiles", 1l);
       blkSize = conf.getLong("test.nnbench.blocksize", 1l);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Test;
+
+public class TestNNBench extends HadoopTestCase {
+  private static final String BASE_DIR =
+      new File(System.getProperty("test.build.data", "build/test/data"),
+          "NNBench").getAbsolutePath();
+
+  public TestNNBench() throws IOException {
+    super(LOCAL_MR, LOCAL_FS, 1, 1);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    getFileSystem().delete(new Path(BASE_DIR), true);
+    getFileSystem().delete(new Path(NNBench.DEFAULT_RES_FILE_NAME), true);
+    super.tearDown();
+  }
+
+  @Test(timeout = 30000)
+  public void testNNBenchCreateReadAndDelete() throws Exception {
+    runNNBench(createJobConf(), "create_write");
+    Path path = new Path(BASE_DIR + "/data/file_0_0");
+    assertTrue("create_write should create the file",
+        getFileSystem().exists(path));
+    runNNBench(createJobConf(), "open_read");
+    runNNBench(createJobConf(), "delete");
+    assertFalse("Delete operation should delete the file",
+        getFileSystem().exists(path));
+  }
+
+  @Test(timeout = 30000)
+  public void testNNBenchCreateAndRename() throws Exception {
+    runNNBench(createJobConf(), "create_write");
+    Path path = new Path(BASE_DIR + "/data/file_0_0");
+    assertTrue("create_write should create the file",
+        getFileSystem().exists(path));
+    runNNBench(createJobConf(), "rename");
+    Path renamedPath = new Path(BASE_DIR + "/data/file_0_r_0");
+    assertFalse("Rename should rename the file", getFileSystem().exists(path));
+    assertTrue("Rename should rename the file",
+        getFileSystem().exists(renamedPath));
+  }
+
+  private void runNNBench(Configuration conf, String operation)
+      throws Exception {
+    String[] genArgs = { "-operation", operation, "-baseDir", BASE_DIR,
+        "-startTime", "" + (Time.now() / 1000 + 3) };
+
+    assertEquals(0, ToolRunner.run(conf, new NNBench(), genArgs));
+  }
+
+}