YARN-6625. yarn application -list returns a tracking URL for AM that doesn't work in secured and HA environment. (Yufei Gu)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java

@@ -23,10 +23,12 @@ import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.net.HttpURLConnection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.Collection;
 
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -38,12 +40,12 @@ import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
 import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.util.RMHAUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,7 +68,8 @@ public class AmIpFilter implements Filter {
   private String[] proxyHosts;
   private Set<String> proxyAddresses = null;
   private long lastUpdate;
-  private Map<String, String> proxyUriBases;
+  @VisibleForTesting
+  Map<String, String> proxyUriBases;
 
   @Override
   public void init(FilterConfig conf) throws ServletException {
@@ -187,24 +190,56 @@ public class AmIpFilter implements Filter {
     }
   }
 
-  protected String findRedirectUrl() throws ServletException {
-    String addr;
-    if (proxyUriBases.size() == 1) {  // external proxy or not RM HA
+  @VisibleForTesting
+  public String findRedirectUrl() throws ServletException {
+    String addr = null;
+    if (proxyUriBases.size() == 1) {
+      // external proxy or not RM HA
       addr = proxyUriBases.values().iterator().next();
-    } else {                          // RM HA
+    } else {
+      // RM HA
       YarnConfiguration conf = new YarnConfiguration();
-      String activeRMId = RMHAUtils.findActiveRMHAId(conf);
-      String addressPropertyPrefix = YarnConfiguration.useHttps(conf)
-          ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
-          : YarnConfiguration.RM_WEBAPP_ADDRESS;
-      String host = conf.get(
-          HAUtil.addSuffix(addressPropertyPrefix, activeRMId));
-      addr = proxyUriBases.get(host);
+      for (String rmId : getRmIds(conf)) {
+        String url = getUrlByRmId(conf, rmId);
+        if (isValidUrl(url)) {
+          addr = url;
+          break;
+        }
+      }
     }
+
     if (addr == null) {
       throw new ServletException(
           "Could not determine the proxy server for redirection");
     }
     return addr;
   }
+
+  @VisibleForTesting
+  Collection<String> getRmIds(YarnConfiguration conf) {
+    return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
+  }
+
+  @VisibleForTesting
+  String getUrlByRmId(YarnConfiguration conf, String rmId) {
+    String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
+        YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
+        YarnConfiguration.RM_WEBAPP_ADDRESS;
+    String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
+    return proxyUriBases.get(host);
+  }
+
+  @VisibleForTesting
+  public boolean isValidUrl(String url) {
+    boolean isValid = false;
+    try {
+      HttpURLConnection conn =
+          (HttpURLConnection) new URL(url).openConnection();
+      conn.connect();
+      isValid = conn.getResponseCode() == HttpURLConnection.HTTP_OK;
+    } catch (Exception e) {
+      LOG.debug("Failed to connect to " + url + ": " + e.toString());
+    }
+    return isValid;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java

@@ -18,25 +18,42 @@
 
 package org.apache.hadoop.yarn.server.webproxy.amfilter;
 
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.net.HttpURLConnection;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import javax.servlet.*;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import static org.junit.Assert.*;
-
-import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
-import org.junit.Test;
-import org.mockito.Mockito;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * Test AmIpFilter. Requests to a no declared hosts should has way through
@@ -121,6 +138,36 @@ public class TestAmFilter {
     filter.destroy();
   }
 
+  @Test
+  public void testFindRedirectUrl() throws Exception {
+    final String rm1 = "rm1";
+    final String rm2 = "rm2";
+
+    final String rm1Url = "host1:8088";
+    final String rm2Url = "host2:8088";
+
+    TestAmIpFilter filter = new TestAmIpFilter();
+    TestAmIpFilter spy = Mockito.spy(filter);
+    // make sure findRedirectUrl() go to HA branch
+    spy.proxyUriBases = new HashMap<>();
+    spy.proxyUriBases.put(rm1, rm1Url);
+    spy.proxyUriBases.put(rm2, rm2Url);
+
+    Collection<String> rmIds = new ArrayList<>(Arrays.asList(rm1, rm2));
+    Mockito.doReturn(rmIds).when(spy).getRmIds(
+        Mockito.any(YarnConfiguration.class));
+    Mockito.doReturn(rm1Url).when(spy).getUrlByRmId(
+        Mockito.any(YarnConfiguration.class), Mockito.eq(rm2));
+    Mockito.doReturn(rm2Url).when(spy)
+        .getUrlByRmId(Mockito.any(YarnConfiguration.class), Mockito.eq(rm1));
+
+    // Stub "isValidUrl" and returns false for rm1, returns true for rm2.
+    Mockito.doReturn(false).when(spy).isValidUrl(Mockito.eq(rm1Url));
+    Mockito.doReturn(true).when(spy).isValidUrl(Mockito.eq(rm2Url));
+
+    assertEquals(spy.findRedirectUrl(), rm2Url);
+  }
+
   /**
    * Test AmIpFilter
    */