YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan.

Reviewed-by: Inigo Goiri <inigoiri@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java

@@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.ReservationRequests;
 import org.apache.hadoop.yarn.api.records.ReservationRequest;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo;
@@ -33,7 +36,10 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.hadoop.yarn.server.router.webapp.TestFederationInterceptorREST.getReservationSubmissionRequestInfo;
 import static org.junit.Assert.assertEquals;
@@ -122,4 +128,37 @@ public class TestRouterServerUtil {
         "definitionInfo Or ReservationRequests is Null.",
         () -> RouterServerUtil.convertReservationDefinition(definitionInfo3));
   }
+
+  @Test
+  public void testLoadFederationPolicyManager() throws Exception {
+
+    // In this unit test, we have configured the yarn-site.xml file with
+    // the yarn.federation.policy-manager-params parameter,
+    // and subsequently, we parse this parameter.
+    // We have configured two subclusters, SC-1 and SC-2,
+    // with routerPolicyWeights set to SC-1:0.7 and SC-2:0.3,
+    // and amrmPolicyWeights set to SC-1:0.6 and SC-2:0.4.
+    // Additionally, headroomAlpha is set to 1.0.
+
+    YarnConfiguration conf = new YarnConfiguration();
+    String defaultPolicyParamString = conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS,
+        YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+    assertNotNull(defaultPolicyParamString);
+    ByteBuffer defaultPolicyParam = ByteBuffer.wrap(
+        defaultPolicyParamString.getBytes(StandardCharsets.UTF_8));
+    WeightedPolicyInfo policyInfo = WeightedPolicyInfo.fromByteBuffer(defaultPolicyParam);
+    float headroomAlpha = policyInfo.getHeadroomAlpha();
+    Map<SubClusterIdInfo, Float> routerPolicyWeights = policyInfo.getRouterPolicyWeights();
+    Map<SubClusterIdInfo, Float> amrmPolicyWeights = policyInfo.getAMRMPolicyWeights();
+
+    SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1");
+    SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2");
+
+    assertEquals(1.0, headroomAlpha, 0.001);
+    assertEquals(0.7, routerPolicyWeights.get(sc1), 0.001);
+    assertEquals(0.3, routerPolicyWeights.get(sc2), 0.001);
+
+    assertEquals(0.6, amrmPolicyWeights.get(sc1), 0.001);
+    assertEquals(0.4, amrmPolicyWeights.get(sc2), 0.001);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml

@@ -39,4 +39,8 @@
     <name>yarn.resourcemanager.resource-profiles.source-file</name>
     <value>profiles/sample-profiles-1.json</value>
   </property>
+  <property>
+    <name>yarn.federation.policy-manager-params</name>
+    <value>{"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"}</value>
+  </property>
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md

@@ -235,6 +235,26 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
 |`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. |
 |`yarn.federation.machine-list` | `<path of machine-list file>` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example: <br/> <br/> node1, subcluster1, rack1 <br/> node2, subcluster2, rack1 <br/> node3, subcluster3, rack2 <br/> node4, subcluster3, rack2 |
 
+- yarn.federation.policy-manager-params
+
+  To configure the `yarn.federation.policy-manager-params` parameter, which represents the weight policy for the default queue,
+  and where the relevant information will be parsed as `WeightedPolicyInfo`.
+
+  We can use the following JSON format for configuration:
+
+  ```xml
+      <property>
+         <name>yarn.federation.policy-manager-params</name>
+         <value>{"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"}</value>
+      </property>
+  ```
+
+  This JSON configuration allows you to define the weight policy for default queue, where:
+
+  - The `routerPolicyWeights` section specifies the weightings for router policies. For instance, with a weight of `0.3` assigned to `SC-2` and `0.7` assigned to `SC-1`, this configuration will allocate `30%` of submitted application requests to `SC-2` and `70%` to `SC-1`.
+  - The `amrmPolicyWeights` represents the allocation ratios for Application Master when request containers from different subclusters' RM. For instance, when an AM requests containers, it will request `40%` of the containers from `SC-2` and `60%` of the containers from `SC-1`.
+  - The `headroomAlpha` used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load.
+
 How to configure the policy-manager
 --------------------