HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Contributed by Varun Saxena)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED
     HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job
     (Akshay Radia via brandonli)
 
+    HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via
+    Arpit Agarwal)
+
   OPTIMIZATIONS
 
     HADOOP-11323. WritableComparator#compare keeps reference to byte array.

hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml

@@ -30,4 +30,11 @@
     <Bug pattern="REC_CATCH_EXCEPTION"/>
     <Bug code="REC"/>
   </Match>
+
+  <!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code -->
+  <Match>
+    <Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/>
+    <Field name="returnValue"/>
+    <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+  </Match>
 </FindBugsFilter>

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java

@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -97,7 +98,9 @@ class CompressionEmulationUtil {
   
   private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE = 
     new CompressionRatioLookupTable();
-  
+
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   /**
    * This is a {@link Mapper} implementation for generating random text data.
    * It uses {@link RandomTextDataGenerator} for generating text data and the
@@ -133,7 +136,8 @@ class CompressionEmulationUtil {
         String randomKey = rtg.getRandomWord();
         String randomValue = rtg.getRandomWord();
         context.write(new Text(randomKey), new Text(randomValue));
-        bytes -= (randomValue.getBytes().length + randomKey.getBytes().length);
+        bytes -= (randomValue.getBytes(charsetUTF8).length +
+            randomKey.getBytes(charsetUTF8).length);
       }
     }
   }

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java

@@ -41,6 +41,7 @@ import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -112,6 +113,8 @@ class DistributedCacheEmulator {
 
   Configuration conf; // gridmix configuration
 
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   // Pseudo local file system where local FS based distributed cache files are
   // created by gridmix.
   FileSystem pseudoLocalFs = null;
@@ -436,9 +439,10 @@ class DistributedCacheEmulator {
     for (Iterator it = dcFiles.iterator(); it.hasNext();) {
       Map.Entry entry = (Map.Entry)it.next();
       LongWritable fileSize =
-          new LongWritable(Long.valueOf(entry.getValue().toString()));
+          new LongWritable(Long.parseLong(entry.getValue().toString()));
       BytesWritable filePath =
-          new BytesWritable(entry.getKey().toString().getBytes());
+          new BytesWritable(
+          entry.getKey().toString().getBytes(charsetUTF8));
 
       byteCount += fileSize.get();
       bytesSync += fileSize.get();
@@ -515,7 +519,7 @@ class DistributedCacheEmulator {
             // local FS based distributed cache file.
             // Create this file on the pseudo local FS.
             String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString();
-            long fileSize = Long.valueOf(fileSizes[i]);
+            long fileSize = Long.parseLong(fileSizes[i]);
             Path mappedLocalFilePath =
                 PseudoLocalFs.generateFilePath(fileId, fileSize)
                     .makeQualified(pseudoLocalFs.getUri(),

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.mapred.gridmix;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
@@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob {
    */
   static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
 
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   public GenerateDistCacheData(Configuration conf) throws IOException {
     super(conf, 0L, JOB_NAME);
   }
@@ -152,7 +155,8 @@ class GenerateDistCacheData extends GridmixJob {
     public void map(LongWritable key, BytesWritable value, Context context)
         throws IOException, InterruptedException {
 
-      String fileName = new String(value.getBytes(), 0, value.getLength());
+      String fileName = new String(value.getBytes(), 0,
+          value.getLength(), charsetUTF8);
       Path path = new Path(fileName);
 
       FSDataOutputStream dos =

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java

@@ -729,40 +729,40 @@ public class Gridmix extends Configured implements Tool {
     out.println("   -users <usersResourceURI> : URI that contains the users list.");
     out.println("Configuration parameters:");
     out.println("   General parameters:");
-    out.printf("       %-48s : Output directory\n", GRIDMIX_OUT_DIR);
-    out.printf("       %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
-    out.printf("       %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
-    out.printf("       %-48s : User resolution class\n", GRIDMIX_USR_RSV);
-    out.printf("       %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
+    out.printf("       %-48s : Output directory%n", GRIDMIX_OUT_DIR);
+    out.printf("       %-48s : Submitting threads%n", GRIDMIX_SUB_THR);
+    out.printf("       %-48s : Queued job desc%n", GRIDMIX_QUE_DEP);
+    out.printf("       %-48s : User resolution class%n", GRIDMIX_USR_RSV);
+    out.printf("       %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
     out.println("   Parameters related to job submission:");    
-    out.printf("       %-48s : Default queue\n",
+    out.printf("       %-48s : Default queue%n",
         GridmixJob.GRIDMIX_DEFAULT_QUEUE);
-    out.printf("       %-48s : Enable/disable using queues in trace\n",
+    out.printf("       %-48s : Enable/disable using queues in trace%n",
         GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
-    out.printf("       %-48s : Job submission policy (%s)\n",
+    out.printf("       %-48s : Job submission policy (%s)%n",
         GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
     out.println("   Parameters specific for LOADJOB:");
-    out.printf("       %-48s : Key fraction of rec\n",
+    out.printf("       %-48s : Key fraction of rec%n",
         AvgRecordFactory.GRIDMIX_KEY_FRC);
     out.println("   Parameters specific for SLEEPJOB:");
-    out.printf("       %-48s : Whether to ignore reduce tasks\n",
+    out.printf("       %-48s : Whether to ignore reduce tasks%n",
         SleepJob.SLEEPJOB_MAPTASK_ONLY);
-    out.printf("       %-48s : Number of fake locations for map tasks\n",
+    out.printf("       %-48s : Number of fake locations for map tasks%n",
         JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
-    out.printf("       %-48s : Maximum map task runtime in mili-sec\n",
+    out.printf("       %-48s : Maximum map task runtime in mili-sec%n",
         SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
-    out.printf("       %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
+    out.printf("       %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n",
         SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
     out.println("   Parameters specific for STRESS submission throttling policy:");
-    out.printf("       %-48s : jobs vs task-tracker ratio\n",
+    out.printf("       %-48s : jobs vs task-tracker ratio%n",
         StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
-    out.printf("       %-48s : maps vs map-slot ratio\n",
+    out.printf("       %-48s : maps vs map-slot ratio%n",
         StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
-    out.printf("       %-48s : reduces vs reduce-slot ratio\n",
+    out.printf("       %-48s : reduces vs reduce-slot ratio%n",
         StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
-    out.printf("       %-48s : map-slot share per job\n",
+    out.printf("       %-48s : map-slot share per job%n",
         StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
-    out.printf("       %-48s : reduce-slot share per job\n",
+    out.printf("       %-48s : reduce-slot share per job%n",
         StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
    }
 

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java

@@ -124,7 +124,7 @@ class PseudoLocalFs extends FileSystem {
     } else {
       String[] parts = path.toUri().getPath().split("\\.");
       try {
-        fileSize = Long.valueOf(parts[parts.length - 1]);
+        fileSize = Long.parseLong(parts[parts.length - 1]);
         valid = (fileSize >= 0);
       } catch (NumberFormatException e) {
         valid = false;

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java

@@ -119,7 +119,23 @@ public class Statistics implements Component<Statistics.JobStats> {
     }
     return new JobStats(maps, reds, job);
   }
-  
+
+  private static void addToNumMapsSubmitted(int numMaps) {
+    numMapsSubmitted += numMaps;
+  }
+
+  private static void addToNumReducesSubmitted(int numReduces) {
+    numReducesSubmitted += numReduces;
+  }
+
+  private static void subtractFromNumMapsSubmitted(int numMaps) {
+    numMapsSubmitted -= numMaps;
+  }
+
+  private static void subtractFromNumReducesSubmitted(int numReduces) {
+    numReducesSubmitted -= numReduces;
+  }
+
   /**
    * Add a submitted job for monitoring.
    */
@@ -131,8 +147,8 @@ public class Statistics implements Component<Statistics.JobStats> {
       return;
     }
     submittedJobsMap.put(seq, stats);
-    numMapsSubmitted += stats.getNoOfMaps();
-    numReducesSubmitted += stats.getNoOfReds();
+    addToNumMapsSubmitted(stats.getNoOfMaps());
+    addToNumReducesSubmitted(stats.getNoOfReds());
   }
 
   /**
@@ -156,8 +172,8 @@ public class Statistics implements Component<Statistics.JobStats> {
     }
     
     // update the total number of submitted map/reduce task count
-    numMapsSubmitted -= stat.getNoOfMaps();
-    numReducesSubmitted -= stat.getNoOfReds();
+    subtractFromNumMapsSubmitted(stat.getNoOfMaps());
+    subtractFromNumReducesSubmitted(stat.getNoOfReds());
     
     completedJobsInCurrentInterval++;
     //check if we have reached the maximum level of job completions.