AMBARI-25433: Adding VDF fails with paywalled repos/urls (#3512)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java

@@ -24,16 +24,25 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.net.URLConnection;
+import java.security.KeyManagementException;
 import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
 
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
@@ -289,12 +298,49 @@ public class URLStreamProvider implements StreamProvider {
     return cookies + "; " + newCookie;
   }
 
+  public static class TrustAllHostnameVerifier implements HostnameVerifier
+  {
+    public boolean verify(String hostname, SSLSession session) { return true; }
+  }
+
+  public static class TrustAllManager implements X509TrustManager
+  {
+    public X509Certificate[] getAcceptedIssuers()
+    {
+      return new X509Certificate[0];
+    }
+    public void checkClientTrusted(X509Certificate[] certs, String authType) {}
+    public void checkServerTrusted(X509Certificate[] certs, String authType) {}
+  }
 
   // ----- helper methods ----------------------------------------------------
 
   // Get a connection
   protected HttpURLConnection getConnection(URL url) throws IOException {
-    return (HttpURLConnection) url.openConnection();
+    URLConnection connection = url.openConnection();
+
+    if (!setupTruststoreForHttps) {
+      HttpsURLConnection httpsConnection = (HttpsURLConnection) connection;
+
+      // Create a trust manager that does not validate certificate chains
+      TrustManager[] trustAllCerts = new TrustManager[] {
+          new TrustAllManager()
+      };
+
+      // Ignore differences between given hostname and certificate hostname
+      HostnameVerifier hostnameVerifier = new TrustAllHostnameVerifier();
+      // Install the all-trusting trust manager
+      try {
+        SSLContext sc = SSLContext.getInstance("SSL");
+        sc.init(null, trustAllCerts, new SecureRandom());
+        httpsConnection.setSSLSocketFactory(sc.getSocketFactory());
+        httpsConnection.setHostnameVerifier(hostnameVerifier);
+      } catch (NoSuchAlgorithmException | KeyManagementException e) {
+        throw new IllegalStateException("Cannot create unverified ssl context.", e);
+      }
+    }
+
+    return (HttpURLConnection) connection;
   }
 
   // Get an ssl connection

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java

@@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.internal;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -580,6 +581,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
       } else {
         URLStreamProvider provider = new URLStreamProvider(connectTimeout, readTimeout,
             ComponentSSLConfiguration.instance());
+        provider.setSetupTruststoreForHttps(false);
 
         stream = provider.readFrom(definitionUrl);
       }
@@ -614,7 +616,17 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
 
     entity.setStack(stackEntity);
 
-    List<RepositoryInfo> repos = holder.xml.repositoryInfo.getRepositories();
+    String credentials = null;
+    if (holder.url != null) {
+      try {
+        URI uri = new URI(holder.url);
+        credentials = uri.getUserInfo();
+      } catch (URISyntaxException e) {
+        throw new AmbariException(String.format("Could not parse url %s", holder.url), e);
+      }
+    }
+
+    List<RepositoryInfo> repos = holder.xml.repositoryInfo.getRepositories(credentials);
 
     StackInfo stack = s_metaInfo.get().getStack(stackId);
 

ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java

@@ -22,6 +22,8 @@ import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -34,12 +36,15 @@ import javax.xml.bind.annotation.XmlTransient;
 import org.apache.ambari.server.stack.Validable;
 import org.apache.ambari.server.state.RepositoryInfo;
 
+import com.google.common.base.Strings;
+
 /**
  * Represents the repository file <code>$STACK_VERSION/repos/repoinfo.xml</code>.
  */
 @XmlRootElement(name="reposinfo")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class RepositoryXml implements Validable{
+  private static final Pattern HTTP_URL_PROTOCOL_PATTERN = Pattern.compile("((http(s)*:\\/\\/))");
 
   @XmlElement(name="latest")
   private String latestUri;
@@ -219,6 +224,16 @@ public class RepositoryXml implements Validable{
    * @return the list of repositories consumable by the web service.
    */
   public List<RepositoryInfo> getRepositories() {
+    return getRepositories(null);
+  }
+
+  /**
+   * @param credentials string with column separated username and password to be inserted in basurl.
+   *                    If set to null baseurl is not changed.
+   *
+   * @return the list of repositories consumable by the web service.
+   */
+  public List<RepositoryInfo> getRepositories(String credentials) {
     List<RepositoryInfo> repos = new ArrayList<>();
 
     for (RepositoryXml.Os o : getOses()) {
@@ -227,7 +242,15 @@ public class RepositoryXml implements Validable{
         for (RepositoryXml.Repo r : o.getRepos()) {
 
           RepositoryInfo ri = new RepositoryInfo();
-          ri.setBaseUrl(r.getBaseUrl());
+          String baseUrl = r.getBaseUrl();
+
+          // add credentials from VDF url to baseurl.
+          if (!Strings.isNullOrEmpty(credentials)) {
+            Matcher matcher = HTTP_URL_PROTOCOL_PATTERN.matcher(baseUrl);
+            baseUrl = matcher.replaceAll("$1" + credentials + "@");
+          }
+
+          ri.setBaseUrl(baseUrl);
           ri.setDefaultBaseUrl(r.getBaseUrl());
           ri.setMirrorsList(r.getMirrorsList());
           ri.setOsType(os.trim());