AMBARI-18216. Both NameNodes reporting version mismatches before Finalizing RU (ncole)

ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java

@@ -23,6 +23,7 @@ import java.net.MalformedURLException;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -39,6 +40,7 @@ import org.apache.ambari.server.state.UpgradeState;
 import org.apache.ambari.server.utils.HTTPUtils;
 import org.apache.ambari.server.utils.HostAndPort;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -144,7 +146,14 @@ public class MasterHostResolver {
               hostsType.master = pair.containsKey(Status.ACTIVE) ? pair.get(Status.ACTIVE) :  null;
               hostsType.secondary = pair.containsKey(Status.STANDBY) ? pair.get(Status.STANDBY) :  null;
             } else {
-              hostsType.master = componentHosts.iterator().next();
+              // !!! we KNOW we have 2 componentHosts if we're here.
+              Iterator<String> iterator = componentHosts.iterator();
+              hostsType.master = iterator.next();
+              hostsType.secondary = iterator.next();
+
+              LOG.warn("Could not determine the active/standby states from NameNodes {}. " +
+                  "Using {} as active and {} as standby.",
+                  StringUtils.join(componentHosts, ','), hostsType.master, hostsType.secondary);
             }
           }
           break;

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java

@@ -409,6 +409,9 @@ public class UpgradeHelper {
 
                   builder.add(context, hostsType, service.serviceName,
                       svc.isClientOnlyService(), pc, null);
+                } else {
+                  LOG.warn("Could not orchestrate NameNode.  Hosts could not be resolved: hosts={}, active={}, standby={}",
+                      StringUtils.join(hostsType.hosts, ','), hostsType.master, hostsType.secondary);
                 }
                 break;
               case NON_ROLLING:

ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java

@@ -17,9 +17,6 @@
  */
 package org.apache.ambari.server.utils;
 
-import org.apache.ambari.server.controller.internal.URLStreamProvider;
-import org.apache.ambari.server.proxy.ProxyService;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -29,6 +26,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.ambari.server.proxy.ProxyService;
+
 /**
  * Static Helper methods for HTTP requests.
  */
@@ -46,7 +47,8 @@ public class HTTPUtils {
     String url = urlToRead;
 
     try {
-      URLStreamProvider urlStreamProvider = new URLStreamProvider(ProxyService.URL_CONNECT_TIMEOUT, ProxyService.URL_READ_TIMEOUT, null, null, null);
+      URLStreamProvider urlStreamProvider = new URLStreamProvider(ProxyService.URL_CONNECT_TIMEOUT,
+          ProxyService.URL_READ_TIMEOUT, ComponentSSLConfiguration.instance());
 
       Map<String, List<String>> headers = new HashMap<String, List<String>>();
 

ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java

@@ -1598,6 +1598,74 @@ public class UpgradeHelperTest {
     assertTrue(ht.hosts.contains("h1"));
   }
 
+  @Test
+  public void testResolverBadJmx() throws Exception {
+    Clusters clusters = injector.getInstance(Clusters.class);
+    ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class);
+
+    String clusterName = "c1";
+    String version = "2.1.1.0-1234";
+
+    StackId stackId = new StackId("HDP-2.1.1");
+    clusters.addCluster(clusterName, stackId);
+    Cluster c = clusters.getCluster(clusterName);
+
+    helper.getOrCreateRepositoryVersion(stackId,
+        c.getDesiredStackVersion().getStackVersion());
+
+    c.createClusterVersion(stackId,
+        c.getDesiredStackVersion().getStackVersion(), "admin",
+        RepositoryVersionState.INSTALLING);
+
+    for (int i = 0; i < 2; i++) {
+      String hostName = "h" + (i+1);
+      clusters.addHost(hostName);
+      Host host = clusters.getHost(hostName);
+
+      Map<String, String> hostAttributes = new HashMap<String, String>();
+      hostAttributes.put("os_family", "redhat");
+      hostAttributes.put("os_release_version", "6");
+
+      host.setHostAttributes(hostAttributes);
+
+      host.persist();
+      clusters.mapHostToCluster(hostName, clusterName);
+    }
+
+    // Add services
+    c.addService(serviceFactory.createNew(c, "HDFS"));
+
+    Service s = c.getService("HDFS");
+    ServiceComponent sc = s.addServiceComponent("NAMENODE");
+    sc.addServiceComponentHost("h1");
+    sc.addServiceComponentHost("h2");
+
+    List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE");
+    assertEquals(2, schs.size());
+
+    setConfigMocks();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.internal.nameservices")).andReturn("ha").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes();
+
+    // Notice that these names are all caps.
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes();
+    replay(m_configHelper);
+
+    MasterHostResolver mhr = new BadMasterHostResolver(m_configHelper, c, version);
+
+    HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE");
+    assertNotNull(ht.master);
+    assertNotNull(ht.secondary);
+    assertEquals(2, ht.hosts.size());
+
+    // Should be stored in lowercase.
+    assertTrue(ht.hosts.contains("h1"));
+    assertTrue(ht.hosts.contains("h2"));
+  }
+
+
   /**
    * Tests that advanced {@link Grouping} instances like {@link StopGrouping}
    * work with rolling upgrade packs.
@@ -1743,10 +1811,6 @@ public class UpgradeHelperTest {
    */
   private class MockMasterHostResolver extends MasterHostResolver {
 
-    public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster) {
-      this(configHelper, cluster, null);
-    }
-
     public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) {
       super(configHelper, cluster, version);
     }
@@ -1786,4 +1850,18 @@ public class UpgradeHelperTest {
       binder.bind(ConfigHelper.class).toInstance(m_configHelper);
     }
   }
+
+  private static class BadMasterHostResolver extends MasterHostResolver {
+
+    public BadMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) {
+      super(configHelper, cluster, version);
+    }
+
+    @Override
+    protected String queryJmxBeanValue(String hostname, int port, String beanName,
+        String attributeName, boolean asQuery, boolean encrypted) {
+      return null;
+    }
+
+  }
 }