YARN-10779. Add option to disable lowercase conversion in GetApplicationsRequestPBImpl and ApplicationSubmissionContextPBImpl. Contributed by Peter Bacsko

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1954,7 +1954,10 @@ public class YarnConfiguration extends Configuration {
           false;
   public static final String APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST =
           APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".username.whitelist";
-
+  public static final String APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
+      APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".force-lowercase";
+  public static final boolean DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
+          true;
   /** Enable switch for container log monitoring. */
   public static final String NM_CONTAINER_LOG_MONITOR_ENABLED =
       NM_PREFIX + "container-log-monitor.enable";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java

@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
+
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
@@ -25,6 +28,7 @@ import java.util.Set;
 import org.apache.commons.lang3.Range;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -33,12 +37,14 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
 import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
-
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 @Private
 @Unstable
 public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
+  private static volatile Boolean forceLowerCaseTags;
+
   GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance();
   GetApplicationsRequestProto.Builder builder = null;
   boolean viaProto = false;
@@ -56,11 +62,23 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
 
   public GetApplicationsRequestPBImpl() {
     builder = GetApplicationsRequestProto.newBuilder();
+    initLowerCaseConfig();
   }
 
   public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) {
     this.proto = proto;
     viaProto = true;
+    initLowerCaseConfig();
+  }
+
+  private static void initLowerCaseConfig() {
+    if (forceLowerCaseTags == null) {
+      Configuration conf = new Configuration();
+
+      forceLowerCaseTags =
+          conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
+              DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
+    }
   }
 
   public synchronized GetApplicationsRequestProto getProto() {
@@ -210,7 +228,8 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
     // Convert applicationTags to lower case and add
     this.applicationTags = new HashSet<String>();
     for (String tag : tags) {
-      this.applicationTags.add(StringUtils.toLowerCase(tag));
+      this.applicationTags.add(
+          forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
     }
   }
 
@@ -406,4 +425,9 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
   public String toString() {
     return TextFormat.shortDebugString(getProto());
   }
+
+  @VisibleForTesting
+  static void setForceLowerCaseTags(boolean convert) {
+    GetApplicationsRequestPBImpl.forceLowerCaseTags = convert;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java

@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -29,6 +32,7 @@ import java.util.TreeSet;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -50,13 +54,15 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
-
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 @Private
 @Unstable
 public class ApplicationSubmissionContextPBImpl 
 extends ApplicationSubmissionContext {
+  private static volatile Boolean forceLowerCaseTags;
+
   ApplicationSubmissionContextProto proto = 
       ApplicationSubmissionContextProto.getDefaultInstance();
   ApplicationSubmissionContextProto.Builder builder = null;
@@ -75,14 +81,26 @@ extends ApplicationSubmissionContext {
 
   public ApplicationSubmissionContextPBImpl() {
     builder = ApplicationSubmissionContextProto.newBuilder();
+    initLowerCaseConfig();
   }
 
   public ApplicationSubmissionContextPBImpl(
       ApplicationSubmissionContextProto proto) {
     this.proto = proto;
     viaProto = true;
+    initLowerCaseConfig();
   }
-  
+
+  private static void initLowerCaseConfig() {
+    if (forceLowerCaseTags == null) {
+      Configuration conf = new Configuration();
+
+      forceLowerCaseTags =
+          conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
+              DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
+    }
+  }
+
   public synchronized ApplicationSubmissionContextProto getProto() {
       mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
@@ -287,7 +305,8 @@ extends ApplicationSubmissionContext {
     // Convert applicationTags to lower case and add
     this.applicationTags = new TreeSet<>();
     for (String tag : tags) {
-      this.applicationTags.add(StringUtils.toLowerCase(tag));
+      this.applicationTags.add(
+          forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
     }
   }
 
@@ -719,4 +738,9 @@ extends ApplicationSubmissionContext {
     this.schedulingProperties.clear();
     this.schedulingProperties.putAll(schedulingPropertyMap);
   }
+
+  @VisibleForTesting
+  static void setForceLowerCaseTags(boolean convert) {
+    ApplicationSubmissionContextPBImpl.forceLowerCaseTags = convert;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -4773,4 +4773,10 @@
     <name>yarn.resourcemanager.application.max-tag.length</name>
     <value>100</value>
   </property>
+
+  <property>
+    <description>Specifies whether application tags should be converted to lowercase or not.</description>
+    <name>yarn.resourcemanager.application-tag-based-placement.force-lowercase</name>
+    <value>true</value>
+  </property>
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestGetApplicationsRequestPBImpl {
+  @Parameter
+  @SuppressWarnings("checkstyle:visibilitymodifier")
+  public GetApplicationsRequestPBImpl impl;
+
+  @Test
+  public void testAppTagsLowerCaseConversionDefault() {
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertEquals(s, s.toLowerCase()));
+  }
+
+  @Test
+  public void testAppTagsLowerCaseConversionDisabled() {
+    GetApplicationsRequestPBImpl.setForceLowerCaseTags(false);
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertNotEquals(s, s.toLowerCase()));
+  }
+
+  @Test
+  public void testAppTagsLowerCaseConversionEnabled() {
+    GetApplicationsRequestPBImpl.setForceLowerCaseTags(true);
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertEquals(s, s.toLowerCase()));
+  }
+
+  @Parameters
+  public static Collection<Object[]> data() {
+    List<Object[]> list = new ArrayList<>();
+    list.add(new Object[] {new GetApplicationsRequestPBImpl()});
+    list.add(new Object[] {new GetApplicationsRequestPBImpl(
+        GetApplicationsRequestProto.newBuilder().build())});
+
+    return list;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestApplicationSubmissionContextPBImpl {
+  @Parameter
+  @SuppressWarnings("checkstyle:visibilitymodifier")
+  public ApplicationSubmissionContextPBImpl impl;
+
+  @Test
+  public void testAppTagsLowerCaseConversionDefault() {
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertEquals(s, s.toLowerCase()));
+  }
+
+  @Test
+  public void testAppTagsLowerCaseConversionDisabled() {
+    ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(false);
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertNotEquals(s, s.toLowerCase()));
+  }
+
+  @Test
+  public void testAppTagsLowerCaseConversionEnabled() {
+    ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(true);
+    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    impl.getApplicationTags().forEach(s ->
+        assertEquals(s, s.toLowerCase()));
+  }
+
+  @Parameters
+  public static Collection<Object[]> data() {
+    List<Object[]> list = new ArrayList<>();
+    list.add(new Object[] {new ApplicationSubmissionContextPBImpl()});
+    list.add(new Object[] {new ApplicationSubmissionContextPBImpl(
+        ApplicationSubmissionContextProto.newBuilder().build())});
+
+    return list;
+  }
+}