YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes, checkstyles. (asuresh)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

@@ -230,10 +230,10 @@ public class RMContainerAllocator extends RMContainerRequestor
     // Init startTime to current time. If all goes well, it will be reset after
     // first attempt to contact RM.
     retrystartTime = System.currentTimeMillis();
-    this.scheduledRequests.setNumOpportunisticMapsPer100(
-        conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE,
-            MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE));
-    LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() +
+    this.scheduledRequests.setNumOpportunisticMapsPercent(
+        conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT,
+            MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT));
+    LOG.info(this.scheduledRequests.getNumOpportunisticMapsPercent() +
         "% of the mappers will be scheduled using OPPORTUNISTIC containers");
   }
 
@@ -1026,14 +1026,14 @@ public class RMContainerAllocator extends RMContainerRequestor
     final Map<TaskAttemptId, ContainerRequest> maps =
       new LinkedHashMap<TaskAttemptId, ContainerRequest>();
     int mapsMod100 = 0;
-    int numOpportunisticMapsPer100 = 0;
+    int numOpportunisticMapsPercent = 0;
 
-    void setNumOpportunisticMapsPer100(int numMaps) {
-      this.numOpportunisticMapsPer100 = numMaps;
+    void setNumOpportunisticMapsPercent(int numMaps) {
+      this.numOpportunisticMapsPercent = numMaps;
     }
 
-    int getNumOpportunisticMapsPer100() {
-      return this.numOpportunisticMapsPer100;
+    int getNumOpportunisticMapsPercent() {
+      return this.numOpportunisticMapsPercent;
     }
 
     @VisibleForTesting
@@ -1080,7 +1080,7 @@ public class RMContainerAllocator extends RMContainerRequestor
         maps.put(event.getAttemptID(), request);
         addContainerReq(request);
       } else {
-        if (mapsMod100 < numOpportunisticMapsPer100) {
+        if (mapsMod100 < numOpportunisticMapsPercent) {
           request =
               new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP,
                   mapNodeLabelExpression);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -988,9 +988,9 @@ public interface MRJobConfig {
    * requested by the AM will be opportunistic. If the total number of maps
    * for the job is less than 'x', then ALL maps will be OPPORTUNISTIC
    */
-  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE =
-      "mapreduce.job.num-opportunistic-maps-percentage";
-  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0;
+  public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT =
+      "mapreduce.job.num-opportunistic-maps-percent";
+  public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
 
   /**
    * A comma-separated list of properties whose value will be redacted.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java

@@ -145,7 +145,7 @@ public class TestMROpportunisticMaps {
     job.setNumReduceTasks(numReducers);
 
     // All OPPORTUNISTIC
-    job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent);
+    job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, percent);
     job.setInt("mapreduce.map.maxattempts", 1);
     job.setInt("mapreduce.reduce.maxattempts", 1);
     job.setInt("mapred.test.num_lines", numLines);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.api.records;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 
 /**
@@ -37,5 +38,6 @@ public enum ContainerState {
   COMPLETE,
 
   /** Scheduled (awaiting resources) at the NM. */
+  @InterfaceStability.Unstable
   SCHEDULED
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -299,6 +299,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Setting that controls whether opportunistic container allocation
    *  is enabled or not. */
+  @Unstable
   public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED =
       RM_PREFIX + "opportunistic-container-allocation.enabled";
   public static final boolean
@@ -306,12 +307,14 @@ public class YarnConfiguration extends Configuration {
 
   /** Number of nodes to be used by the Opportunistic Container allocator for
    * dispatching containers during container allocation. */
+  @Unstable
   public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
       RM_PREFIX + "opportunistic-container-allocation.nodes-used";
   public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
       10;
 
   /** Frequency for computing least loaded NMs. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS =
       RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
   public static final long
@@ -319,6 +322,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Comparator for determining node load for scheduling of opportunistic
    * containers. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
       RM_PREFIX + "nm-container-queuing.load-comparator";
   public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
@@ -326,6 +330,7 @@ public class YarnConfiguration extends Configuration {
 
   /** Value of standard deviation used for calculation of queue limit
    * thresholds. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV =
       RM_PREFIX + "nm-container-queuing.queue-limit-stdev";
   public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV =
@@ -334,6 +339,7 @@ public class YarnConfiguration extends Configuration {
   /** Min length of container queue at NodeManager. This is a cluster-wide
    * configuration that acts as the lower-bound of optimal queue length
    * calculated by the NodeQueueLoadMonitor */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH =
       RM_PREFIX + "nm-container-queuing.min-queue-length";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5;
@@ -341,17 +347,20 @@ public class YarnConfiguration extends Configuration {
   /** Max length of container queue at NodeManager. This is a cluster-wide
    * configuration that acts as the upper-bound of optimal queue length
    * calculated by the NodeQueueLoadMonitor */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH =
       RM_PREFIX + "nm-container-queuing.max-queue-length";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15;
 
   /** Min queue wait time for a container at a NodeManager. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
       RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
       10;
 
   /** Max queue wait time for a container queue at a NodeManager. */
+  @Unstable
   public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
       RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
   public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java

@@ -87,7 +87,6 @@ import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
 import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
 
 import com.google.common.annotations.VisibleForTesting;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java

@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md

@@ -72,10 +72,10 @@ By default, allocation of opportunistic containers is performed centrally throug
 
 The following command can be used to run a sample pi map-reduce job, executing 40% of mappers using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version of Hadoop you are using):
 ```
-$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100
+$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percent="40" 50 100
 ```
 
-By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers.
+By changing the value of `mapreduce.job.num-opportunistic-maps-percent` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers.
 
 
 ###<a name="Opportunistic_Containers_in_Web_UI"></a>Opportunistic Containers in Web UI