YARN-10373. Create Matchers for CS mapping rules. Contributed by Gergely Pollak

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+public interface MappingRuleMatcher {
+  /**
+   * Returns true if the matcher matches the current context.
+   * @param variables The variable context, which contains all the variables
+   * @return true if this matcher matches to the provided variable set
+   */
+  boolean match(VariableContext variables);
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java

@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import java.util.Arrays;
+
+/**
+ * This class contains all the matcher and some helper methods to generate them.
+ */
+public class MappingRuleMatchers {
+  /**
+   * Utility class, hiding constructor.
+   */
+  private MappingRuleMatchers() {}
+
+  /**
+   * MatchAllMatcher is a matcher which matches everything.
+   */
+  public static class MatchAllMatcher implements MappingRuleMatcher {
+    /**
+     * The match will return true in all cases, to match all submissions.
+     * @param variables The variable context, which contains all the variables
+     * @return true
+     */
+    @Override
+    public boolean match(VariableContext variables) {
+      return true;
+    }
+
+    @Override
+    public String toString() {
+      return "MatchAllMatcher";
+    }
+  }
+
+  /**
+   * VariableMatcher will check if a provided variable's value matches the
+   * provided value. The provided value might contain variables as well, which
+   * will get evaluated before the comparison.
+   */
+  public static class VariableMatcher implements MappingRuleMatcher {
+    /**
+     * Name of the variable to be checked.
+     */
+    private String variable;
+    /**
+     * The value which should match the variable's value.
+     */
+    private String value;
+
+    VariableMatcher(String variable, String value) {
+      this.variable = variable;
+      this.value = value == null ? "" : value;
+    }
+
+    /**
+     * The method will replace all variables in the value, then compares this
+     * substituted value against the variable's value, if they match we return
+     * true.
+     * If the variable is null we always return false.
+     * @param variables The variable context, which contains all the variables
+     * @return true if the value matches the variable's value, false otherwise
+     */
+    @Override
+    public boolean match(VariableContext variables) {
+      if (variable == null) {
+        return false;
+      }
+
+      String substituted = variables.replaceVariables(value);
+      return substituted.equals(variables.get(variable));
+    }
+
+    @Override
+    public String toString() {
+      return "VariableMatcher{" +
+        "variable='" + variable + '\'' +
+        ", value='" + value + '\'' +
+        '}';
+    }
+  }
+
+  /**
+   * AndMatcher is a basic boolean matcher which takes multiple other
+   * matcher as it's arguments, and on match it checks if all of them are true.
+   */
+  public static class AndMatcher implements MappingRuleMatcher {
+    /**
+     * The list of matchers to be checked during evaluation.
+     */
+    private MappingRuleMatcher[] matchers;
+
+    /**
+     * Constructor.
+     * @param matchers List of matchers to be checked during evaluation
+     */
+    AndMatcher(MappingRuleMatcher...matchers) {
+      this.matchers = matchers;
+    }
+
+    /**
+     * This match method will go through all the provided matchers and call
+     * their match method, if all match we return true.
+     * @param variables The variable context, which contains all the variables
+     * @return true if all matchers match
+     */
+    @Override
+    public boolean match(VariableContext variables) {
+      for (MappingRuleMatcher matcher : matchers) {
+        if (!matcher.match(variables)) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    @Override
+    public String toString() {
+      return "AndMatcher{" +
+          "matchers=" + Arrays.toString(matchers) +
+          '}';
+    }
+  }
+
+  /**
+   * OrMatcher is a basic boolean matcher which takes multiple other
+   * matcher as its arguments, and on match it checks if any of them are true.
+   */
+  public static class OrMatcher implements MappingRuleMatcher {
+    /**
+     * The list of matchers to be checked during evaluation.
+     */
+    private MappingRuleMatcher[] matchers;
+
+    /**
+     * Constructor.
+     * @param matchers List of matchers to be checked during evaluation
+     */
+    OrMatcher(MappingRuleMatcher...matchers) {
+      this.matchers = matchers;
+    }
+
+    /**
+     * This match method will go through all the provided matchers and call
+     * their match method, if any of them match we return true.
+     * @param variables The variable context, which contains all the variables
+     * @return true if any of the matchers match
+     */
+    @Override
+    public boolean match(VariableContext variables) {
+      for (MappingRuleMatcher matcher : matchers) {
+        if (matcher.match(variables)) {
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    @Override
+    public String toString() {
+      return "OrMatcher{" +
+          "matchers=" + Arrays.toString(matchers) +
+          '}';
+    }
+  }
+
+  /**
+   * Convenience method to create a variable matcher which matches against the
+   * username.
+   * @param userName The username to be matched
+   * @return VariableMatcher with %user as the variable
+   */
+  public static MappingRuleMatcher createUserMatcher(String userName) {
+    return new VariableMatcher("%user", userName);
+  }
+
+  /**
+   * Convenience method to create a variable matcher which matches against the
+   * user's primary group.
+   * @param groupName The groupName to be matched
+   * @return VariableMatcher with %primary_group as the variable
+   */
+  public static MappingRuleMatcher createGroupMatcher(String groupName) {
+    return new VariableMatcher("%primary_group", groupName);
+  }
+
+  /**
+   * Convenience method to create a composite matcher which matches against the
+   * user's user name and the user's primary group. Only matches if both
+   * matches.
+   * @param userName The username to be matched
+   * @param groupName The groupName to be matched
+   * @return AndMatcher with two matchers one for userName and one for
+   * primaryGroup
+   */
+  public static MappingRuleMatcher createUserGroupMatcher(
+      String userName, String groupName) {
+    return new AndMatcher(
+        createUserMatcher(userName),
+        createGroupMatcher(groupName));
+  }
+
+  /**
+   * Convenience method to create a variable matcher which matches against the
+   * submitted application's name.
+   * @param name The name to be matched
+   * @return VariableMatcher with %application as the variable
+   */
+  public static MappingRuleMatcher createApplicationNameMatcher(String name) {
+    return new VariableMatcher("%application", name);
+  }
+
+
+  /**
+   * Convenience method to create a matcher that matches all
+   * @return MatchAllMatcher.
+   */
+  public static MappingRuleMatcher createAllMatcher() {
+    return new MatchAllMatcher();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java

@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestMappingRuleMatchers extends TestCase {
+
+  @Test
+  public void testCatchAll() {
+    VariableContext variables = new VariableContext();
+    MappingRuleMatcher matcher = new MappingRuleMatchers.MatchAllMatcher();
+
+    assertTrue(matcher.match(variables));
+  }
+
+  @Test
+  public void testVariableMatcher() {
+    VariableContext matchingContext = new VariableContext();
+    matchingContext.put("%user", "bob");
+    matchingContext.put("%primary_group", "developers");
+    matchingContext.put("%application", "TurboMR");
+    matchingContext.put("%custom", "Matching string");
+
+    VariableContext mismatchingContext = new VariableContext();
+    mismatchingContext.put("%user", "dave");
+    mismatchingContext.put("%primary_group", "testers");
+    mismatchingContext.put("%application", "Tester APP");
+    mismatchingContext.put("%custom", "Not matching string");
+
+    VariableContext emptyContext = new VariableContext();
+
+    Map<String, MappingRuleMatcher> matchers = new HashMap<>();
+    matchers.put("User matcher", MappingRuleMatchers.createUserMatcher("bob"));
+    matchers.put("Group matcher",
+        MappingRuleMatchers.createGroupMatcher("developers"));
+    matchers.put("Application name matcher",
+        MappingRuleMatchers.createApplicationNameMatcher("TurboMR"));
+    matchers.put("Custom matcher",
+        new MappingRuleMatchers.VariableMatcher("%custom", "Matching string"));
+
+    matchers.forEach((matcherName, matcher) -> {
+      assertTrue(matcherName + " with matchingContext should match",
+          matcher.match(matchingContext));
+      assertFalse(matcherName + " with mismatchingContext shouldn't match",
+          matcher.match(mismatchingContext));
+      assertFalse(matcherName + " with emptyContext shouldn't match",
+          matcher.match(emptyContext));
+    });
+  }
+
+  @Test
+  public void testVariableMatcherSubstitutions() {
+    VariableContext matchingContext = new VariableContext();
+    matchingContext.put("%user", "bob");
+    matchingContext.put("%primary_group", "developers");
+    matchingContext.put("%application", "TurboMR");
+    matchingContext.put("%custom", "bob");
+    matchingContext.put("%cus", "b");
+    matchingContext.put("%tom", "ob");
+
+    VariableContext mismatchingContext = new VariableContext();
+    mismatchingContext.put("%user", "dave");
+    mismatchingContext.put("%primary_group", "testers");
+    mismatchingContext.put("%application", "Tester APP");
+    mismatchingContext.put("%custom", "bob");
+    mismatchingContext.put("%cus", "b");
+    mismatchingContext.put("%tom", "ob");
+
+    MappingRuleMatcher customUser =
+        new MappingRuleMatchers.VariableMatcher("%custom", "%user");
+
+    MappingRuleMatcher userCusTom =
+        new MappingRuleMatchers.VariableMatcher("%user", "%cus%tom");
+
+    MappingRuleMatcher userCustom =
+        new MappingRuleMatchers.VariableMatcher("%user", "%custom");
+
+    MappingRuleMatcher userUser =
+        new MappingRuleMatchers.VariableMatcher("%user", "%user");
+
+    MappingRuleMatcher userStatic =
+        new MappingRuleMatchers.VariableMatcher("%user", "bob");
+
+    assertTrue("%custom should match %user in matching context",
+        customUser.match(matchingContext));
+    assertTrue("%user should match %custom in matching context",
+        userCustom.match(matchingContext));
+    assertTrue("%user (bob) should match %cus%tom (b + ob) in matching context",
+        userCusTom.match(matchingContext));
+    assertTrue("%user should match %user in any context",
+        userUser.match(matchingContext));
+    assertTrue("%user (bob) should match bob in in matching context",
+        userStatic.match(matchingContext));
+
+    assertFalse(
+        "%custom (bob) should NOT match %user (dave) in mismatching context",
+        customUser.match(mismatchingContext));
+    assertFalse(
+        "%user (dave) should NOT match %custom (bob) in mismatching context",
+        userCustom.match(mismatchingContext));
+    assertFalse(
+        "%user (dave) should NOT match %cus%tom (b+ob) in mismatching context",
+        userCusTom.match(mismatchingContext));
+    assertTrue("%user should match %user in any context",
+        userUser.match(mismatchingContext));
+    assertFalse(
+        "%user (dave) should NOT match match bob in in matching context",
+        userStatic.match(mismatchingContext));
+  }
+
+  @Test
+  public void testVariableMatcherWithEmpties() {
+    VariableContext emptyContext = new VariableContext();
+    VariableContext allNull = new VariableContext();
+    allNull.put("%null", null);
+    allNull.put("%empty", null);
+
+    VariableContext allEmpty = new VariableContext();
+    allEmpty.put("%null", "");
+    allEmpty.put("%empty", "");
+
+    VariableContext allMakesSense = new VariableContext();
+    allMakesSense.put("%null", null);
+    allMakesSense.put("%empty", "");
+
+    VariableContext allFull = new VariableContext();
+    allFull.put("%null", "full");
+    allFull.put("%empty", "full");
+
+    MappingRuleMatcher nullMatcher =
+        new MappingRuleMatchers.VariableMatcher("%null", null);
+
+    MappingRuleMatcher emptyMatcher =
+        new MappingRuleMatchers.VariableMatcher("%empty", "");
+
+    //nulls should be handled as empty strings, so all should match
+    assertTrue(nullMatcher.match(emptyContext));
+    assertTrue(emptyMatcher.match(emptyContext));
+    assertTrue(nullMatcher.match(allEmpty));
+    assertTrue(emptyMatcher.match(allEmpty));
+    assertTrue(nullMatcher.match(allNull));
+    assertTrue(emptyMatcher.match(allNull));
+    assertTrue(nullMatcher.match(allMakesSense));
+    assertTrue(emptyMatcher.match(allMakesSense));
+
+    //neither null nor "" should match an actual string
+    assertFalse(nullMatcher.match(allFull));
+    assertFalse(emptyMatcher.match(allFull));
+
+    MappingRuleMatcher nullVarMatcher =
+        new MappingRuleMatchers.VariableMatcher(null, "");
+
+    //null variable should never match
+    assertFalse(nullVarMatcher.match(emptyContext));
+    assertFalse(nullVarMatcher.match(allNull));
+    assertFalse(nullVarMatcher.match(allEmpty));
+    assertFalse(nullVarMatcher.match(allMakesSense));
+    assertFalse(nullVarMatcher.match(allFull));
+  }
+
+  @Test
+  public void testBoolOperatorMatchers() {
+    VariableContext developerBob = new VariableContext();
+    developerBob.put("%user", "bob");
+    developerBob.put("%primary_group", "developers");
+
+
+    VariableContext testerBob = new VariableContext();
+    testerBob.put("%user", "bob");
+    testerBob.put("%primary_group", "testers");
+
+    VariableContext testerDave = new VariableContext();
+    testerDave.put("%user", "dave");
+    testerDave.put("%primary_group", "testers");
+
+
+    VariableContext accountantDave = new VariableContext();
+    accountantDave.put("%user", "dave");
+    accountantDave.put("%primary_group", "accountants");
+
+    MappingRuleMatcher userBob =
+        new MappingRuleMatchers.VariableMatcher("%user", "bob");
+
+    MappingRuleMatcher groupDevelopers =
+        new MappingRuleMatchers.VariableMatcher(
+           "%primary_group", "developers");
+
+    MappingRuleMatcher groupAccountants =
+        new MappingRuleMatchers.VariableMatcher(
+            "%primary_group", "accountants");
+
+    MappingRuleMatcher developerBobMatcher = new MappingRuleMatchers.AndMatcher(
+            userBob, groupDevelopers);
+
+    MappingRuleMatcher testerDaveMatcher =
+        MappingRuleMatchers.createUserGroupMatcher("dave", "testers");
+
+    MappingRuleMatcher accountantOrBobMatcher =
+        new MappingRuleMatchers.OrMatcher(groupAccountants, userBob);
+
+    assertTrue(developerBobMatcher.match(developerBob));
+    assertFalse(developerBobMatcher.match(testerBob));
+    assertFalse(developerBobMatcher.match(testerDave));
+    assertFalse(developerBobMatcher.match(accountantDave));
+
+    assertFalse(testerDaveMatcher.match(developerBob));
+    assertFalse(testerDaveMatcher.match(testerBob));
+    assertTrue(testerDaveMatcher.match(testerDave));
+    assertFalse(testerDaveMatcher.match(accountantDave));
+
+    assertTrue(accountantOrBobMatcher.match(developerBob));
+    assertTrue(accountantOrBobMatcher.match(testerBob));
+    assertFalse(accountantOrBobMatcher.match(testerDave));
+    assertTrue(accountantOrBobMatcher.match(accountantDave));
+  }
+
+  @Test
+  public void testToStrings() {
+    MappingRuleMatcher var = new MappingRuleMatchers.VariableMatcher("%a", "b");
+    MappingRuleMatcher all = MappingRuleMatchers.createAllMatcher();
+    MappingRuleMatcher and = new MappingRuleMatchers.AndMatcher(var, all, var);
+    MappingRuleMatcher or = new MappingRuleMatchers.OrMatcher(var, all, var);
+
+    assertEquals("VariableMatcher{variable='%a', value='b'}", var.toString());
+    assertEquals("MatchAllMatcher", all.toString());
+    assertEquals("AndMatcher{matchers=[" + var.toString() +
+        ", " + all.toString() +
+        ", " + var.toString() + "]}", and.toString());
+    assertEquals("OrMatcher{matchers=[" + var.toString() +
+        ", " + all.toString() +
+        ", " + var.toString() + "]}", or.toString());
+  }
+
+}