YARN-10177: Backport YARN-7307 to branch-2.10 Allow client/AM update supported resource types via YARN APIs

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -204,4 +205,23 @@ public abstract class RegisterApplicationMasterResponse {
   @Unstable
   public abstract void setSchedulerResourceTypes(
       EnumSet<SchedulerResourceTypes> types);
+
+  /**
+   * Get available resource types supported by RM.
+   *
+   * @return a Map of RM settings
+   */
+  @Public
+  @Unstable
+  public abstract List<ResourceTypeInfo> getResourceTypes();
+
+  /**
+   * Set the resource types used by RM.
+   *
+   * @param types
+   *          a set of the resource types supported by RM.
+   */
+  @Private
+  @Unstable
+  public abstract void setResourceTypes(List<ResourceTypeInfo> types);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -220,6 +220,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
         ResourceTypes.COUNTABLE, minRes, maxRes);
   }
 
+  public static ResourceInformation newInstance(String name, String units,
+      ResourceTypes resourceType) {
+    return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L,
+        Long.MAX_VALUE);
+  }
+
   public static ResourceInformation newInstance(String name, long value) {
     return ResourceInformation
         .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -989,6 +989,17 @@ public class YarnConfiguration extends Configuration {
    */
   public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
 
+  /**
+   * Enable/disable loading resource-types.xml at client side.
+   */
+  @Public
+  @Unstable
+  public static final String YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = YARN_PREFIX
+      + "client.load.resource-types.from-server";
+  @Public
+  @Unstable
+  public static final boolean DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = false;
+
   /**
    * Timeout in seconds for YARN node graceful decommission.
    * This is the maximal time to wait for running containers and applications

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -655,4 +653,25 @@ public class ResourceUtils {
 
     return info;
   }
+
+  /**
+   * Reinitialize all resource types from external source (in case of client,
+   * server will send the updated list and local resourceutils cache will be
+   * updated as per server's list of resources)
+   *
+   * @param resourceTypeInfo
+   *          List of resource types
+   */
+  public static void reinitializeResources(
+      List<ResourceTypeInfo> resourceTypeInfo) {
+    Map<String, ResourceInformation> resourceInformationMap = new HashMap<>();
+
+    for (ResourceTypeInfo resourceType : resourceTypeInfo) {
+      resourceInformationMap.put(resourceType.getName(),
+          ResourceInformation.newInstance(resourceType.getName(),
+              resourceType.getDefaultUnit(), resourceType.getResourceType()));
+    }
+    ResourceUtils
+        .initializeResourcesFromResourceInformationMap(resourceInformationMap);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto

@@ -48,6 +48,7 @@ message RegisterApplicationMasterResponseProto {
   optional string queue = 5;
   repeated NMTokenProto nm_tokens_from_previous_attempts = 6;
   repeated SchedulerResourceTypes scheduler_resource_types = 7;
+  repeated ResourceTypeInfoProto resource_types = 9;
 }
 
 message FinishApplicationMasterRequestProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -103,6 +104,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
 import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
 import org.apache.hadoop.yarn.api.records.Token;
@@ -123,6 +125,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -146,6 +149,7 @@ public class YarnClientImpl extends YarnClient {
   String timelineDTRenewer;
   private boolean timelineV1ServiceEnabled;
   protected boolean timelineServiceBestEffort;
+  private boolean loadResourceTypesFromServer;
 
   private static final String ROOT = "root";
 
@@ -191,6 +195,11 @@ public class YarnClientImpl extends YarnClient {
     timelineServiceBestEffort = conf.getBoolean(
         YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
+
+    loadResourceTypesFromServer = conf.getBoolean(
+        YarnConfiguration.YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER,
+        YarnConfiguration.DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER);
+
     super.serviceInit(conf);
   }
 
@@ -209,6 +218,13 @@ public class YarnClientImpl extends YarnClient {
     } catch (IOException e) {
       throw new YarnRuntimeException(e);
     }
+
+    // Reinitialize local resource types cache from list of resources pulled from
+    // RM.
+    if (loadResourceTypesFromServer) {
+      ResourceUtils.reinitializeResources(getResourceTypeInfo());
+    }
+
     super.serviceStart();
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java

@@ -18,10 +18,8 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
+import com.google.protobuf.ByteString;
+import com.google.protobuf.TextFormat;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
@@ -29,21 +27,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
 import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.TextFormat;
-
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 @Private
 @Unstable
@@ -59,6 +64,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
   private List<Container> containersFromPreviousAttempts = null;
   private List<NMToken> nmTokens = null;
   private EnumSet<SchedulerResourceTypes> schedulerResourceTypes = null;
+  private List<ResourceTypeInfo> resourceTypeInfo = null;
 
   public RegisterApplicationMasterResponsePBImpl() {
     builder = RegisterApplicationMasterResponseProto.newBuilder();
@@ -123,9 +129,11 @@ public class RegisterApplicationMasterResponsePBImpl extends
     if(schedulerResourceTypes != null) {
       addSchedulerResourceTypes();
     }
+    if (resourceTypeInfo != null) {
+      addResourceTypeInfosToProto();
+    }
   }
 
-
   private void maybeInitBuilder() {
     if (viaProto || builder == null) {
       builder = RegisterApplicationMasterResponseProto.newBuilder(proto);
@@ -456,4 +464,75 @@ public class RegisterApplicationMasterResponsePBImpl extends
   private NMToken convertFromProtoFormat(NMTokenProto proto) {
     return new NMTokenPBImpl(proto);
   }
+
+  private ResourceTypeInfoPBImpl convertFromProtoFormat(
+      ResourceTypeInfoProto p) {
+    return new ResourceTypeInfoPBImpl(p);
+  }
+
+  private ResourceTypeInfoProto convertToProtoFormat(ResourceTypeInfo t) {
+    return ((ResourceTypeInfoPBImpl) t).getProto();
+  }
+
+  @Override
+  public List<ResourceTypeInfo> getResourceTypes() {
+    initResourceTypeInfosList();
+    return this.resourceTypeInfo;
+  }
+
+  @Override
+  public void setResourceTypes(List<ResourceTypeInfo> types) {
+    if (resourceTypeInfo == null) {
+      builder.clearResourceTypes();
+    }
+    this.resourceTypeInfo = types;
+  }
+
+  private void addResourceTypeInfosToProto() {
+    maybeInitBuilder();
+    builder.clearResourceTypes();
+    if (resourceTypeInfo == null) {
+      return;
+    }
+    Iterable<ResourceTypeInfoProto> iterable = new Iterable<ResourceTypeInfoProto>() {
+      @Override
+      public Iterator<ResourceTypeInfoProto> iterator() {
+        return new Iterator<ResourceTypeInfoProto>() {
+
+          Iterator<ResourceTypeInfo> iter = resourceTypeInfo.iterator();
+
+          @Override
+          public boolean hasNext() {
+            return iter.hasNext();
+          }
+
+          @Override
+          public ResourceTypeInfoProto next() {
+            return convertToProtoFormat(iter.next());
+          }
+
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException();
+
+          }
+        };
+
+      }
+    };
+    builder.addAllResourceTypes(iterable);
+  }
+
+  private void initResourceTypeInfosList() {
+    if (this.resourceTypeInfo != null) {
+      return;
+    }
+    RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+    List<ResourceTypeInfoProto> list = p.getResourceTypesList();
+    resourceTypeInfo = new ArrayList<ResourceTypeInfo>();
+
+    for (ResourceTypeInfoProto a : list) {
+      resourceTypeInfo.add(convertFromProtoFormat(a));
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -3632,4 +3632,13 @@
     <name>yarn.resourcemanager.application.max-tag.length</name>
     <value>100</value>
   </property>
+
+  <property>
+    <description>
+      Provides an option for client to load supported resource types from RM
+      instead of depending on local resource-types.xml file.
+    </description>
+    <name>yarn.client.load.resource-types.from-server</name>
+    <value>false</value>
+  </property>
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java

@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.io.IOException;
@@ -181,6 +182,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
 
     response.setSchedulerResourceTypes(getScheduler()
         .getSchedulingResourceTypes());
+    response.setResourceTypes(ResourceUtils.getResourcesTypeInfo());
   }
 
   @Override