AMBARI-5095. Added latest repository url lookup (ncole)

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -31,6 +31,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
 
 import javax.xml.bind.JAXBException;
 
@@ -50,6 +53,7 @@ import org.apache.ambari.server.state.RepositoryInfo;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.Stack;
 import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.LatestRepoCallable;
 import org.apache.ambari.server.state.stack.MetricDefinition;
 import org.apache.ambari.server.state.stack.RepositoryXml;
 import org.apache.ambari.server.state.stack.RepositoryXml.Os;
@@ -108,6 +112,7 @@ public class AmbariMetaInfo {
   private static final List<String> ALL_SUPPORTED_OS = Arrays.asList(
       "centos5", "redhat5", "centos6", "redhat6", "oraclelinux5",
       "oraclelinux6", "suse11", "sles11", "ubuntu12");
+  
   private final ActionDefinitionManager adManager = new ActionDefinitionManager();
   private String serverVersion = "undefined";
   private List<StackInfo> stacksResult = new ArrayList<StackInfo>();
@@ -677,6 +682,15 @@ public class AmbariMetaInfo {
         "stackRoot = " + stackRootAbsPath);
     }
 
+    ExecutorService es = Executors.newSingleThreadExecutor(new ThreadFactory() {
+      @Override
+      public Thread newThread(Runnable r) {
+        return new Thread(r, "Stack Version Loading Thread");
+      }
+    });
+    
+    List<LatestRepoCallable> lookupList = new ArrayList<LatestRepoCallable>();
+    
     for (StackInfo stack : stacks) {
       LOG.debug("Adding new stack to known stacks"
         + ", stackName = " + stack.getName()
@@ -698,8 +712,8 @@ public class AmbariMetaInfo {
           + ", stackVersion=" + stack.getVersion()
           + ", repoFolder=" + repositoryFolder.getPath());
 
-        List<RepositoryInfo> repositoryInfoList = getRepository
-          (repositoryFolder, stack.getVersion());
+        List<RepositoryInfo> repositoryInfoList = getRepository(repositoryFolder,
+            stack, lookupList);
 
         stack.getRepositories().addAll(repositoryInfoList);
       } else {
@@ -719,6 +733,10 @@ public class AmbariMetaInfo {
               resolveHooksFolder(stack);
       stack.setStackHooksFolder(stackHooksToUse);
     }
+    
+    es.invokeAll(lookupList);
+    
+    es.shutdown();
   }
 
 
@@ -726,9 +744,10 @@ public class AmbariMetaInfo {
     return serverVersion;
   }
 
-  private List<RepositoryInfo> getRepository(File repositoryFile, String stackVersion)
+  private List<RepositoryInfo> getRepository(File repositoryFile, StackInfo stack,
+      List<LatestRepoCallable> lookupList)
       throws JAXBException {
-
+    
     RepositoryXml rxml = StackExtensionHelper.unmarshal(RepositoryXml.class, repositoryFile);
 
     List<RepositoryInfo> list = new ArrayList<RepositoryInfo>();
@@ -743,10 +762,11 @@ public class AmbariMetaInfo {
           ri.setOsType(os.trim());
           ri.setRepoId(r.getRepoId());
           ri.setRepoName(r.getRepoName());
-
+          ri.setLatestBaseUrl(r.getBaseUrl());
+          
           if (null != metainfoDAO) {
             LOG.debug("Checking for override for base_url");
-            String key = generateRepoMetaKey(r.getRepoName(), stackVersion,
+            String key = generateRepoMetaKey(r.getRepoName(), stack.getVersion(),
                 o.getType(), r.getRepoId(), REPOSITORY_XML_PROPERTY_BASEURL);
             MetainfoEntity entity = metainfoDAO.findByKey(key);
             if (null != entity) {
@@ -763,6 +783,11 @@ public class AmbariMetaInfo {
         }
       }
     }
+    
+    if (null != rxml.getLatestURI() && list.size() > 0) {
+      lookupList.add(new LatestRepoCallable(rxml.getLatestURI(),
+          repositoryFile.getParentFile(), stack));
+    }
 
     return list;
 

ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java

@@ -28,16 +28,18 @@ public class RepositoryResponse {
   private String repoName;
   private String mirrorsList;
   private String defaultBaseUrl;
+  private String latestBaseUrl;
   
   
   public RepositoryResponse(String baseUrl, String osType, String repoId,
-      String repoName, String mirrorsList, String defaultBaseUrl) {
+      String repoName, String mirrorsList, String defaultBaseUrl, String latestBaseUrl) {
     setBaseUrl(baseUrl);
     setOsType(osType);
     setRepoId(repoId);
     setRepoName(repoName);
     setMirrorsList(mirrorsList);
     setDefaultBaseUrl(defaultBaseUrl);
+    setLatestBaseUrl(latestBaseUrl);
   }
 
   public String getStackName() {
@@ -109,6 +111,14 @@ public class RepositoryResponse {
   public void setDefaultBaseUrl(String url) {
     this.defaultBaseUrl = url;
   }
+  
+  public String getLatestBaseUrl() {
+    return latestBaseUrl;
+  }
+  
+  public void setLatestBaseUrl(String url) {
+    latestBaseUrl = url;
+  }
 
 
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java

@@ -74,7 +74,10 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
   public static final String REPOSITORY_VERIFY_BASE_URL_PROPERTY_ID = PropertyHelper
       .getPropertyId("Repositories", "verify_base_url");
   
+  public static final String REPOSITORY_LATEST_BASE_URL_PROPERTY_ID = PropertyHelper
+      .getPropertyId("Repositories", "latest_base_url");
 
+  
   private static Set<String> pkPropertyIds = new HashSet<String>(
       Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID,
               STACK_VERSION_PROPERTY_ID, OS_TYPE_PROPERTY_ID,
@@ -162,6 +165,9 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
         
         setResourceProperty(resource, REPOSITORY_DEFAULT_BASE_URL_PROPERTY_ID,
             response.getDefaultBaseUrl(), requestedIds);
+        
+        setResourceProperty(resource, REPOSITORY_LATEST_BASE_URL_PROPERTY_ID,
+            response.getLatestBaseUrl(), requestedIds);
 
         resources.add(resource);
     }

ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java

@@ -27,6 +27,7 @@ public class RepositoryInfo {
   private String repoName;
   private String mirrorsList;
   private String defaultBaseUrl;
+  private String latestBaseUrl;
 
   /**
    * @return the baseUrl
@@ -112,6 +113,20 @@ public class RepositoryInfo {
     defaultBaseUrl = url;
   }
 
+  /**
+   * @return the latest determined base url
+   */
+  public String getLatestBaseUrl() {
+    return latestBaseUrl;
+  }
+  
+  /**
+   * @param url the latest determined base url
+   */
+  public void setLatestBaseUrl(String url) {
+    latestBaseUrl = url;
+  }  
+  
   @Override
   public String toString() {
     return "[ repoInfo: "
@@ -126,6 +141,9 @@ public class RepositoryInfo {
   
   public RepositoryResponse convertToResponse()
   {
-    return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(), getRepoName(), getMirrorsList(), getDefaultBaseUrl());
+    return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(),
+        getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl());
   }
+
+
 }

ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.StackInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+/**
+ * Encapsulates the work to resolve the latest repo information for a stack.
+ * This class must be used AFTER the stack has created its owned repositories.
+ */
+public class LatestRepoCallable implements Callable<Void> {
+  private static final int LOOKUP_CONNECTION_TIMEOUT = 2000;
+  private static final int LOOKUP_READ_TIMEOUT = 1000;
+  
+  private final static Logger LOG = LoggerFactory.getLogger(LatestRepoCallable.class);
+  
+  private String sourceUri = null;
+  private File stackRepoFolder = null;
+  private StackInfo stack = null;
+  
+  public LatestRepoCallable(String latestSourceUri, File stackRepoFolder, StackInfo stack) {
+    this.sourceUri = latestSourceUri;
+    this.stackRepoFolder = stackRepoFolder;
+    this.stack = stack;
+  }
+
+  @Override
+  public Void call() throws Exception {
+    
+    Type type = new TypeToken<Map<String, Map<String, Object>>>(){}.getType();
+    Gson gson = new Gson();
+    
+    Map<String, Map<String, Object>> latestUrlMap = null;
+    
+    try {
+      if (sourceUri.startsWith("http")) {
+        
+        URLStreamProvider streamProvider = new URLStreamProvider(
+            LOOKUP_CONNECTION_TIMEOUT, LOOKUP_READ_TIMEOUT,
+            null, null, null);
+        
+        LOG.info("Loading latest URL info from " + sourceUri);
+        latestUrlMap = gson.fromJson(new InputStreamReader(
+            streamProvider.readFrom(sourceUri)), type);
+      } else {
+        File jsonFile = null;
+        if (sourceUri.charAt(0) == '.') {
+          jsonFile = new File(stackRepoFolder, sourceUri);
+        } else {
+          jsonFile = new File(sourceUri);
+        }
+        
+        if (jsonFile.exists()) {
+          LOG.info("Loading latest URL info from " + jsonFile);
+          latestUrlMap = gson.fromJson(new FileReader(jsonFile), type);
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("Could not load the URI " + sourceUri + " (" + e.getMessage() + ")");
+      throw e;
+    }
+    
+    
+    if (null != latestUrlMap) {
+      for (RepositoryInfo ri : stack.getRepositories()) {
+        if (latestUrlMap.containsKey(ri.getRepoId())) {
+          Map<String, Object> valueMap = latestUrlMap.get(ri.getRepoId());
+          if (valueMap.containsKey("latest")) {
+            
+            @SuppressWarnings("unchecked")
+            Map<String, String> osMap = (Map<String, String>) valueMap.get("latest");
+            if (osMap.containsKey(ri.getOsType())) {
+              ri.setLatestBaseUrl(osMap.get(ri.getOsType()));
+            }
+          }
+        }
+      }
+    }
+    
+    return null;
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java

@@ -27,13 +27,23 @@ import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
 /**
- * Represents the repository file <code>$STACK_VERSION/metainfo.xml</code>.
+ * Represents the repository file <code>$STACK_VERSION/repos/repoinfo.xml</code>.
  */
 @XmlRootElement(name="reposinfo")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class RepositoryXml {
+
+  @XmlElement(name="latest")
+  private String latestUri;
   @XmlElement(name="os")
   private List<Os> oses = new ArrayList<Os>();
+
+  /**
+   * @return the latest URI defined, if any.
+   */
+  public String getLatestURI() {
+    return latestUri;
+  }
   
   /**
    * @return the list of <code>os</code> elements.
@@ -42,7 +52,6 @@ public class RepositoryXml {
     return oses;
   }
   
-  
   /**
    * The <code>os</code> tag.
    */
@@ -77,10 +86,11 @@ public class RepositoryXml {
    */
   @XmlAccessorType(XmlAccessType.FIELD)
   public static class Repo {
-    private String baseurl;
-    private String mirrorslist;
-    private String repoid;
-    private String reponame;
+    private String baseurl = null;
+    private String mirrorslist = null;
+    private String repoid = null;
+    private String reponame = null;
+    private String latest = null;
 
     private Repo() {
     }
@@ -113,6 +123,10 @@ public class RepositoryXml {
       return reponame;
     }
     
+    public String getLatestUri() {
+      return latest;
+    }
+    
     
   }  
   

ambari-server/src/main/resources/properties.json

@@ -182,6 +182,7 @@
         "Repositories/mirrors_list",
         "Repositories/default_base_url",
         "Repositories/verify_base_url",
+        "Repositories/latest_base_url",
         "_"
     ],
     "StackService":[

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -27,7 +27,6 @@ import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.lang.reflect.Method;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -43,7 +42,16 @@ import junit.framework.Assert;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.api.util.StackExtensionHelper;
-import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.CustomCommandDefinition;
+import org.apache.ambari.server.state.DependencyInfo;
+import org.apache.ambari.server.state.OperatingSystemInfo;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.Stack;
+import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.stack.MetricDefinition;
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
@@ -1279,4 +1287,13 @@ public class AmbariMetaInfoTest {
     Assert.assertTrue(found);
   }
 
+  @Test
+  public void testLatestRepo() throws Exception {
+    
+    for (RepositoryInfo ri : metaInfo.getRepositories("HDP", "2.1.1", "centos6")) {
+      Assert.assertEquals("http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.1.1.0-118",
+          ri.getLatestBaseUrl());
+    }
+  }
+
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryResourceProviderTest.java

@@ -57,7 +57,7 @@ public class RepositoryResourceProviderTest {
     AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class);
 
     RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS,
-        VAL_REPO_ID, VAL_REPO_NAME, null, null);
+        VAL_REPO_ID, VAL_REPO_NAME, null, null, null);
     rr.setStackName(VAL_STACK_NAME);
     rr.setStackVersion(VAL_STACK_VERSION);
     Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>();
@@ -128,7 +128,7 @@ public class RepositoryResourceProviderTest {
     AmbariManagementController managementController = EasyMock.createMock(AmbariManagementController.class);
 
     RepositoryResponse rr = new RepositoryResponse(VAL_BASE_URL, VAL_OS,
-        VAL_REPO_ID, VAL_REPO_NAME, null, null);
+        VAL_REPO_ID, VAL_REPO_NAME, null, null ,null);
     Set<RepositoryResponse> allResponse = new HashSet<RepositoryResponse>();
     allResponse.add(rr);    
     

ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/hdp.json

@@ -0,0 +1,10 @@
+{
+  "HDP-2.1.1": {
+    "latest": {
+      "centos6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.1.1.0-118",
+      "redhat6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.1.1.0-118",
+      "oraclelinux6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.1.1.0-118",
+      "suse11": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11/2.x/BUILDS/2.1.1.0-118"
+    }
+  }
+}

ambari-server/src/test/resources/stacks/HDP/2.1.1/repos/repoinfo.xml

@@ -16,45 +16,46 @@
    limitations under the License.
 -->
 <reposinfo>
+  <latest>./hdp.json</latest>
   <os type="centos6">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>
   <os type="centos5">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>
   <os type="redhat6">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>
   <os type="redhat5">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>
   <os type="suse11">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>
   <os type="sles11">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.0.6.0</baseurl>
-      <repoid>HDP-2.0.8</repoid>
+      <repoid>HDP-2.1.1</repoid>
       <reponame>HDP</reponame>
     </repo>
   </os>