AMBARI-13693. Ambari Blueprint sets inconsistent Yarn RM values (DIPAYAN BHOWMICK via srimanth)

contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java

@@ -45,6 +45,11 @@ public class Services {
   private static final String YARN_RESOURCEMANAGER_HA_RM_IDS_KEY = "yarn.resourcemanager.ha.rm-ids";
   private static final String YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.address.";
   private static final String YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.https.address.";
+  private static final String YARN_RESOURCEMANAGER_HOSTNAME_KEY = "yarn.resourcemanager.hostname";
+  private static final String YARN_RESOURCEMANAGER_HOSTNAME_PARTIAL_KEY = YARN_RESOURCEMANAGER_HOSTNAME_KEY + ".";
+  private static final String YARN_RESOURCEMANAGER_DEFAULT_HTTP_PORT = "8088";
+  private static final String YARN_RESOURCEMANAGER_DEFAULT_HTTPS_PORT = "8090";
+
 
   private static final String YARN_ATS_URL = "yarn.timeline-server.url";
   private final static String YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY = "yarn.timeline-service.webapp.address";
@@ -80,10 +85,9 @@ public class Services {
 
   private String getRMUrlFromClusterConfig() {
     String url;
-    String protocol;
 
-    String haEnabled = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HA_ENABLED);
-    String httpPolicy = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_HTTP_POLICY);
+    String haEnabled = getYarnConfig(YARN_RESOURCEMANAGER_HA_ENABLED);
+    String httpPolicy = getYarnConfig(YARN_HTTP_POLICY);
 
     if (!(HTTP_ONLY.equals(httpPolicy) || HTTPS_ONLY.equals(httpPolicy))) {
       LOG.error(String.format("RA030 Unknown value %s of yarn-site/yarn.http.policy. HTTP_ONLY assumed.", httpPolicy));
@@ -94,34 +98,32 @@ public class Services {
       String[] urls = getRMHAUrls(httpPolicy);
       url = getActiveRMUrl(urls);
     } else {
-      if (httpPolicy.equals(HTTPS_ONLY)) {
-        protocol = "https";
-        url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTPS_KEY);
-      } else {
-        protocol = "http";
-        url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTP_KEY);
+      url =  (httpPolicy.equals(HTTPS_ONLY)) ? getYarnConfig(YARN_RESOURCEMANAGER_HTTPS_KEY)
+        : getYarnConfig(YARN_RESOURCEMANAGER_HTTP_KEY);
+
+      if (url == null || url.isEmpty()) {
+        url = getYarnConfig(YARN_RESOURCEMANAGER_HOSTNAME_KEY).trim() + ":" + getDefaultRMPort(httpPolicy);
       }
-      url = addProtocolIfMissing(url, protocol);
+
+      url = addProtocolIfMissing(url, getProtocol(httpPolicy));
     }
     return url;
   }
 
   private String[] getRMHAUrls(String httpPolicy) {
-    String haRmIds = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HA_RM_IDS_KEY);
+    String haRmIds = getYarnConfig(YARN_RESOURCEMANAGER_HA_RM_IDS_KEY);
     String[] ids = haRmIds.split(",");
     int index = 0;
     String[] urls = new String[ids.length];
     for (String id : ids) {
-      String url, protocol;
-      if (HTTPS_ONLY.equals(httpPolicy)) {
-        url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY + id);
-        protocol = "https";
-      } else {
-        url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY + id);
-        protocol = "http";
+      String url = (HTTPS_ONLY.equals(httpPolicy)) ? getYarnConfig(YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY + id)
+        : getYarnConfig(YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY + id);
+
+      if (url == null || url.isEmpty()) {
+        url = getYarnConfig(YARN_RESOURCEMANAGER_HOSTNAME_PARTIAL_KEY + id).trim() + ":" + getDefaultRMPort(httpPolicy);
       }
 
-      urls[index++] = addProtocolIfMissing(url.trim(), protocol);
+      urls[index++] = addProtocolIfMissing(url.trim(), getProtocol(httpPolicy));
     }
     return urls;
   }
@@ -267,9 +269,8 @@ public class Services {
 
   private String getATSUrlFromCluster() {
     String url;
-    String protocol;
 
-    String httpPolicy = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_HTTP_POLICY);
+    String httpPolicy = getYarnConfig(YARN_HTTP_POLICY);
 
     if (!(HTTP_ONLY.equals(httpPolicy) || HTTPS_ONLY.equals(httpPolicy))) {
       LOG.error(String.format("RA030 Unknown value %s of yarn-site/yarn.http.policy. HTTP_ONLY assumed.", httpPolicy));
@@ -277,13 +278,11 @@ public class Services {
     }
 
     if (httpPolicy.equals(HTTPS_ONLY)) {
-      protocol = "https";
-      url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY);
+      url = getYarnConfig(YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY);
     } else {
-      protocol = "http";
-      url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY);
+      url = getYarnConfig(YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY);
     }
-    url = addProtocolIfMissing(url, protocol);
+    url = addProtocolIfMissing(url, getProtocol(httpPolicy));
 
     return url;
   }
@@ -317,4 +316,20 @@ public class Services {
   public static boolean hasProtocol(String url) {
     return url.matches("^[^:]+://.*$");
   }
+
+  private String getProtocol(String yarnHttpPolicyConfig) {
+    return HTTPS_ONLY.equals(yarnHttpPolicyConfig) ? "https" : "http";
+  }
+
+  private String getYarnConfig(String key) {
+    return ambariApi.getCluster().getConfigurationValue(YARN_SITE, key);
+  }
+
+  /**
+   * @param yarnHttpPolicy - The HTTP Policy configured in YARN site file
+   * @return The default resource manager port depending on the http policy
+   */
+  private String getDefaultRMPort(String yarnHttpPolicy) {
+    return (HTTPS_ONLY.equals(yarnHttpPolicy)) ? YARN_RESOURCEMANAGER_DEFAULT_HTTPS_PORT : YARN_RESOURCEMANAGER_DEFAULT_HTTP_PORT;
+  }
 }

contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java

@@ -41,6 +41,8 @@ public class ServicesTest extends EasyMockSupport {
   private static final String HTTP_RM_URL2 = "http://c2.ambari.apache.org:8088";
   private static final String HTTPS_RM_URL1 = "https://c1.ambari.apache.org:8088";
   private static final String HTTPS_RM_URL2 = "https://c2.ambari.apache.org:8088";
+  private static final String RM_URL1_HOST = "c1.ambari.apache.org";
+  private static final String RM_URL2_HOST = "c2.ambari.apache.org";
   private static final String RM_URL1_HOST_PORT = "c1.ambari.apache.org:8088";
   private static final String RM_URL2_HOST_PORT = "c2.ambari.apache.org:8088";
   private static final String RM_INFO_API_ENDPOINT = Services.RM_INFO_API_ENDPOINT;
@@ -273,6 +275,59 @@ public class ServicesTest extends EasyMockSupport {
     assertEquals(HTTP_RM_URL1, services.getRMUrl());
   }
 
+  @Test
+  public void shouldConnectToDefaultHostPortInClusterModeWhenWebaddressConfigIsEmpty() throws Exception {
+    ViewContext viewContext = getViewContext(new HashMap<String, String>());
+    AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+    Cluster cluster = createNiceMock(Cluster.class);
+
+    expect(ambariApi.isClusterAssociated()).andReturn(true).anyTimes();
+    setClusterExpectationWithEmptyWebappConfig(cluster, "HTTP_ONLY");
+    expect(ambariApi.getCluster()).andReturn(cluster).anyTimes();
+
+    Services services = new Services(ambariApi, viewContext);
+
+    replayAll();
+    assertEquals("http://" + RM_URL1_HOST + ":8088", services.getRMUrl());
+
+    reset(cluster);
+    setClusterExpectationWithEmptyWebappConfig(cluster, "HTTPS_ONLY");
+    replay(cluster);
+
+    assertEquals("https://" + RM_URL1_HOST + ":8090", services.getRMUrl());
+
+  }
+
+  @Test
+  public void shouldConnectToDefaultHostPortInClusterModeWithHAWhenWebaddressConfigIsEmpty() throws Exception {
+    ViewContext viewContext = getViewContext(new HashMap<String, String>());
+    AmbariApi ambariApi = createNiceMock(AmbariApi.class);
+    Cluster cluster = createNiceMock(Cluster.class);
+    URLStreamProvider urlStreamProvider = createNiceMock(URLStreamProvider.class);
+
+    expect(ambariApi.isClusterAssociated()).andReturn(true).anyTimes();
+    setClusterExpectationInHAWithEmptyWebappConfig(cluster, "HTTP_ONLY");
+    expect(ambariApi.getCluster()).andReturn(cluster).anyTimes();
+
+    Services services = new Services(ambariApi, viewContext);
+
+    InputStream inputStream = IOUtils.toInputStream("{\"clusterInfo\": {\"haState\": \"ACTIVE\"}}");
+    expect(viewContext.getURLStreamProvider()).andReturn(urlStreamProvider).anyTimes();
+    expect(urlStreamProvider.readFrom(eq("http://" + RM_URL1_HOST + ":8088" + RM_INFO_API_ENDPOINT), eq("GET"), anyString(), EasyMock.<Map<String, String>>anyObject())).andReturn(inputStream);
+
+    replayAll();
+    assertEquals("http://" + RM_URL1_HOST + ":8088", services.getRMUrl());
+
+    reset(cluster, urlStreamProvider);
+    setClusterExpectationInHAWithEmptyWebappConfig(cluster, "HTTPS_ONLY");
+    inputStream = IOUtils.toInputStream("{\"clusterInfo\": {\"haState\": \"ACTIVE\"}}");
+    expect(urlStreamProvider.readFrom(eq("https://" + RM_URL1_HOST + ":8090" + RM_INFO_API_ENDPOINT), eq("GET"), anyString(), EasyMock.<Map<String, String>>anyObject())).andReturn(inputStream);
+    replay(cluster, urlStreamProvider);
+
+    assertEquals("https://" + RM_URL1_HOST + ":8090", services.getRMUrl());
+
+  }
+
   @Test
   public void shouldFetchRMUrlsWhileHAEnabledInClusterMode() throws Exception {
     ViewContext viewContext = getViewContext(new HashMap<String, String>());
@@ -341,6 +396,21 @@ public class ServicesTest extends EasyMockSupport {
     expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.webapp.https.address.rm2")).andReturn(RM_URL2_HOST_PORT);
   }
 
+  private void setClusterExpectationInHAWithEmptyWebappConfig(Cluster cluster, String httpPolicy) {
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.enabled")).andReturn("true");
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.http.policy")).andReturn(httpPolicy);
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.rm-ids")).andReturn("rm1,rm2");
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname.rm1")).andReturn(RM_URL1_HOST);
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname.rm2")).andReturn(RM_URL2_HOST);
+
+  }
+
+  private void setClusterExpectationWithEmptyWebappConfig(Cluster cluster, String httpPolicy) {
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.enabled")).andReturn("false");
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.http.policy")).andReturn(httpPolicy);
+    expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname")).andReturn(RM_URL1_HOST);
+  }
+
   private ViewContext getViewContext(Map<String, String> map) {
     ViewContext viewContextMock = createNiceMock(ViewContext.class);
     expect(viewContextMock.getProperties()).andReturn(map);