YARN-7043. [Partial backport] Cleanup ResourceProfileManager. (wangda)

Change-Id: I463356f37bf1f6a3f1fc3c594c79916e8c0ab913
(cherry picked from commit 39240b61a163b127eec385decd30ffd96e694c28)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java

@@ -52,11 +52,13 @@ public interface ApplicationMasterServiceProcessor {
    * @param request Register Request.
    * @param response Register Response.
    * @throws IOException IOException.
+   * @throws YarnException in critical situation where invalid
+   *         profiles/resources are added.
    */
-  void registerApplicationMaster(
-      ApplicationAttemptId applicationAttemptId,
+  void registerApplicationMaster(ApplicationAttemptId applicationAttemptId,
       RegisterApplicationMasterRequest request,
-      RegisterApplicationMasterResponse response) throws IOException;
+      RegisterApplicationMasterResponse response)
+      throws IOException, YarnException;
 
   /**
    * Allocate call.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java

@@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
 
 /**
  * <p>The protocol between clients and the <code>ResourceManager</code>
@@ -598,8 +599,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
    * </p>
    * @param request request to get the details of a resource profile
    * @return Response containing the details for a particular resource profile
-   * @throws YarnException if resource profiles are not enabled on the RM or
-   *         the profile cannot be found
+   * @throws YarnException if any error happens inside YARN
    * @throws IOException in case of other errors
    */
   @Public

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This exception is thrown when a feature is being used which is not enabled
+ * yet.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class YARNFeatureNotEnabledException extends YarnException {
+  private static final long serialVersionUID = 898023752676L;
+
+  public YARNFeatureNotEnabledException(Throwable cause) {
+    super(cause);
+  }
+
+  public YARNFeatureNotEnabledException(String message) {
+    super(message);
+  }
+
+  public YARNFeatureNotEnabledException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
 import org.apache.hadoop.yarn.conf.ConfigurationProvider;
 import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -37,9 +38,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -561,4 +565,21 @@ public class ResourceUtils {
     }
     return "";
   }
+
+  /**
+   * Get all resource types information from known resource types.
+   * @return List of ResourceTypeInfo
+   */
+  public static List<ResourceTypeInfo> getResourcesTypeInfo() {
+    List<ResourceTypeInfo> array = new ArrayList<>();
+    // Add all resource types
+    Collection<ResourceInformation> resourcesInfo =
+        ResourceUtils.getResourceTypes().values();
+    for (ResourceInformation resourceInfo : resourcesInfo) {
+      array.add(ResourceTypeInfo
+          .newInstance(resourceInfo.getName(), resourceInfo.getUnits(),
+              resourceInfo.getResourceType()));
+    }
+    return array;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java

@@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 
@@ -863,8 +864,7 @@ public abstract class YarnClient extends AbstractService {
    * Get available resource types supported by RM.
    * </p>
    * @return list of supported resource types with detailed information
-   * @throws YarnException if resource profiles are not enabled or the profile
-   *         cannot be found
+   * @throws YarnException if any issue happens inside YARN
    * @throws IOException in case of other others
    */
   @Public

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java

@@ -528,10 +528,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
       long value;
       if (stepFactorValue != 0) {
         value = roundUp
-            ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+            ? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue)
             : roundDown((long) (rValue * by), stepFactorValue);
       } else {
-        value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+        value = roundUp
+            ? (long) Math.ceil((float) (rValue * by))
+            : (long) (rValue * by);
       }
       tmp.setValue(value);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java

@@ -421,6 +421,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
     generateByNewInstance(ApplicationTimeout.class);
     generateByNewInstance(QueueConfigurations.class);
     generateByNewInstance(CollectorInfo.class);
+    generateByNewInstance(ResourceTypeInfo.class);
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java

@@ -82,7 +82,7 @@ class AMSProcessingChain implements ApplicationMasterServiceProcessor {
   public void registerApplicationMaster(
       ApplicationAttemptId applicationAttemptId,
       RegisterApplicationMasterRequest request,
-      RegisterApplicationMasterResponse resp) throws IOException {
+      RegisterApplicationMasterResponse resp) throws IOException, YarnException {
     this.head.registerApplicationMaster(applicationAttemptId, request, resp);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -178,6 +178,7 @@ import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.UTCClock;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 
 
@@ -1806,8 +1807,7 @@ public class ClientRMService extends AbstractService implements
       GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
     GetAllResourceTypeInfoResponse response =
         GetAllResourceTypeInfoResponse.newInstance();
-    response.setResourceTypeInfo(
-        resourceProfilesManager.getAllResourceTypeInfo());
+    response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
     return response;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java

@@ -111,7 +111,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
   public void registerApplicationMaster(
       ApplicationAttemptId applicationAttemptId,
       RegisterApplicationMasterRequest request,
-      RegisterApplicationMasterResponse response) throws IOException {
+      RegisterApplicationMasterResponse response)
+      throws IOException, YarnException {
 
     RMApp app = getRmContext().getRMApps().get(
         applicationAttemptId.getApplicationId());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java

@@ -127,7 +127,8 @@ public class OpportunisticContainerAllocatorAMService
     public void registerApplicationMaster(
         ApplicationAttemptId applicationAttemptId,
         RegisterApplicationMasterRequest request,
-        RegisterApplicationMasterResponse response) throws IOException {
+        RegisterApplicationMasterResponse response)
+        throws IOException, YarnException {
       SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler)
           getScheduler()).getApplicationAttempt(applicationAttemptId);
       if (appAttempt.getOpportunisticContainerContext() == null) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java

@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java

@@ -102,9 +102,11 @@ public class TestApplicationMasterService {
     }
 
     @Override
-    public void registerApplicationMaster(ApplicationAttemptId
-        applicationAttemptId, RegisterApplicationMasterRequest request,
-        RegisterApplicationMasterResponse response) throws IOException {
+    public void registerApplicationMaster(
+        ApplicationAttemptId applicationAttemptId,
+        RegisterApplicationMasterRequest request,
+        RegisterApplicationMasterResponse response)
+        throws IOException, YarnException {
       nextProcessor.registerApplicationMaster(
           applicationAttemptId, request, response);
     }
@@ -144,7 +146,8 @@ public class TestApplicationMasterService {
     public void registerApplicationMaster(
         ApplicationAttemptId applicationAttemptId,
         RegisterApplicationMasterRequest request,
-        RegisterApplicationMasterResponse response) throws IOException {
+        RegisterApplicationMasterResponse response)
+        throws IOException, YarnException {
       beforeRegCount.incrementAndGet();
       nextProcessor.registerApplicationMaster(applicationAttemptId,
               request, response);