AMBARI-7286. Associating repository latest url for deploying cluster using blueprint (Cabir Zounaidou via alejandro)

ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java

@@ -41,13 +41,13 @@ import com.google.gson.reflect.TypeToken;
 public class LatestRepoCallable implements Callable<Void> {
   private static final int LOOKUP_CONNECTION_TIMEOUT = 2000;
   private static final int LOOKUP_READ_TIMEOUT = 1000;
-  
+
   private final static Logger LOG = LoggerFactory.getLogger(LatestRepoCallable.class);
-  
+
   private String sourceUri = null;
   private File stackRepoFolder = null;
   private StackInfo stack = null;
-  
+
   public LatestRepoCallable(String latestSourceUri, File stackRepoFolder, StackInfo stack) {
     this.sourceUri = latestSourceUri;
     this.stackRepoFolder = stackRepoFolder;
@@ -56,19 +56,19 @@ public class LatestRepoCallable implements Callable<Void> {
 
   @Override
   public Void call() throws Exception {
-    
+
     Type type = new TypeToken<Map<String, Map<String, Object>>>(){}.getType();
     Gson gson = new Gson();
-    
+
     Map<String, Map<String, Object>> latestUrlMap = null;
-    
+
     try {
       if (sourceUri.startsWith("http")) {
-        
+
         URLStreamProvider streamProvider = new URLStreamProvider(
             LOOKUP_CONNECTION_TIMEOUT, LOOKUP_READ_TIMEOUT,
             null, null, null);
-        
+
         LOG.info("Loading latest URL info from " + sourceUri);
         latestUrlMap = gson.fromJson(new InputStreamReader(
             streamProvider.readFrom(sourceUri)), type);
@@ -79,7 +79,7 @@ public class LatestRepoCallable implements Callable<Void> {
         } else {
           jsonFile = new File(sourceUri);
         }
-        
+
         if (jsonFile.exists()) {
           LOG.info("Loading latest URL info from " + jsonFile);
           latestUrlMap = gson.fromJson(new FileReader(jsonFile), type);
@@ -89,71 +89,76 @@ public class LatestRepoCallable implements Callable<Void> {
       LOG.error("Could not load the URI " + sourceUri + " (" + e.getMessage() + ")");
       throw e;
     }
-    
-    
+
+
     if (null != latestUrlMap) {
       for (RepositoryInfo ri : stack.getRepositories()) {
         if (latestUrlMap.containsKey(ri.getRepoId())) {
           Map<String, Object> valueMap = latestUrlMap.get(ri.getRepoId());
           if (valueMap.containsKey("latest")) {
-            
+
             @SuppressWarnings("unchecked")
             Map<String, String> osMap = (Map<String, String>) valueMap.get("latest");
-            
+
             String baseUrl = resolveOsUrl(ri.getOsType(), osMap);
             if (null != baseUrl) {
               // !!! in the case where <name>.repo is defined with the base url, strip that off.
               // Agents do the reverse action (take the base url, and append <name>.repo)
-              
+
               String repo_file_format;
               if(ri.getOsType().equals("ubuntu12")) {
                 repo_file_format = "list";
               } else {
                 repo_file_format = "repo";
               }
-              
+
               String repoFileName = stack.getName().toLowerCase() + "." + repo_file_format;
-              int idx = baseUrl.toLowerCase().indexOf(repoFileName); 
-              
+              int idx = baseUrl.toLowerCase().indexOf(repoFileName);
+
               if (-1 != idx && baseUrl.toLowerCase().endsWith(repoFileName)) {
                 baseUrl = baseUrl.substring(0, idx);
               }
-              
+
               if ('/' == baseUrl.charAt(baseUrl.length()-1)) {
                 baseUrl = baseUrl.substring(0, baseUrl.length()-1);
               }
-              
+
               ri.setLatestBaseUrl(baseUrl);
+              if (ri.getBaseUrl() != null
+                  && ri.getBaseUrl().equals(ri.getDefaultBaseUrl())) {
+                // Override baseUrl with the latestBaseUrl.
+                ri.setBaseUrl(baseUrl);
+              }
             }
           }
         }
       }
     }
-    
+
     return null;
   }
-  
+
   /**
    * Resolves a base url given that certain OS types can be used interchangeably.
    * @param os the target os to find
-   * @param osMap the map of os-to-baseurl 
+   * @param osMap the map of os-to-baseurl
    * @return the url for an os.
    */
   private String resolveOsUrl(String os, Map<String, String> osMap) {
-    
+
     // !!! look for the OS directly
     if (osMap.containsKey(os))
       return osMap.get(os);
-    
+
     // !!! os not found, find and return the first compatible one
     Set<String> possibleTypes = OsFamily.findTypes(os);
-    
+
     for (String type : possibleTypes) {
       if (osMap.containsKey(type))
         return osMap.get(type);
     }
-    
+
     return null;
   }
-  
+
 }

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -24,8 +24,6 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
 
 import java.io.File;
 import java.lang.reflect.Method;
@@ -37,6 +35,7 @@ import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 
+import javax.persistence.EntityManager;
 import javax.xml.bind.JAXBException;
 
 import junit.framework.Assert;
@@ -45,6 +44,8 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.api.util.StackExtensionHelper;
 import org.apache.ambari.server.metadata.ActionMetadata;
+import org.apache.ambari.server.orm.GuiceJpaInitializer;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.ComponentInfo;
 import org.apache.ambari.server.state.CustomCommandDefinition;
@@ -108,7 +109,9 @@ public class AmbariMetaInfoTest {
 
   @Before
   public void before() throws Exception {
-    injector = Guice.createInjector(new MockModule());
+    injector = Guice.createInjector(new InMemoryDefaultTestModule());
+    injector.getInstance(GuiceJpaInitializer.class);
+    injector.getInstance(EntityManager.class);
     File stackRoot = new File("src/test/resources/stacks");
     LOG.info("Stacks file " + stackRoot.getAbsolutePath());
     metaInfo = new AmbariMetaInfo(stackRoot, new File("target/version"));
@@ -161,6 +164,130 @@ public class AmbariMetaInfoTest {
     assertFalse(repository.get("centos6").isEmpty());
   }
 
+  @Test
+  public void testGetRepositoryDefault() throws Exception {
+    // Scenario: user has internet and does nothing to repos via api
+    // use the latest
+    String buildDir = tmpFolder.getRoot().getAbsolutePath();
+    AmbariMetaInfo ambariMetaInfo = setupTempAmbariMetaInfo(buildDir);
+    // The current stack already has (HDP, 2.1.1, redhat6) with valid latest
+    // url
+    ambariMetaInfo.init();
+
+    List<RepositoryInfo> redhat6Repo = ambariMetaInfo.getRepositories(
+        STACK_NAME_HDP, "2.1.1", "redhat6");
+    assertNotNull(redhat6Repo);
+    for (RepositoryInfo ri : redhat6Repo) {
+      if (STACK_NAME_HDP.equals(ri.getRepoName())) {
+        assertFalse(ri.getBaseUrl().equals(ri.getDefaultBaseUrl()));
+        assertEquals(ri.getBaseUrl(), ri.getLatestBaseUrl());
+      }
+    }
+  }
+
+  @Test
+  public void testGetRepositoryNoInternetDefault() throws Exception {
+    // Scenario: user has no internet and does nothing to repos via api
+    // use the default
+    String buildDir = tmpFolder.getRoot().getAbsolutePath();
+    AmbariMetaInfo ambariMetaInfo = setupTempAmbariMetaInfo(buildDir);
+    // The current stack already has (HDP, 2.1.1, redhat6).
+
+    // Deleting the json file referenced by the latestBaseUrl to simulate No
+    // Internet.
+    File latestUrlFile = new File(buildDir,
+        "ambari-metaInfo/HDP/2.1.1/repos/hdp.json");
+    FileUtils.deleteQuietly(latestUrlFile);
+    assertTrue(!latestUrlFile.exists());
+    ambariMetaInfo.init();
+
+    List<RepositoryInfo> redhat6Repo = ambariMetaInfo.getRepositories(
+        STACK_NAME_HDP, "2.1.1", "redhat6");
+    assertNotNull(redhat6Repo);
+    for (RepositoryInfo ri : redhat6Repo) {
+      if (STACK_NAME_HDP.equals(ri.getRepoName())) {
+        // baseUrl should be same as defaultBaseUrl since No Internet to load the
+        // latestBaseUrl from the json file.
+        assertEquals(ri.getBaseUrl(), ri.getDefaultBaseUrl());
+      }
+    }
+  }
+
+  @Test
+  public void testGetRepositoryUpdatedBaseUrl() throws Exception {
+    // Scenario: user has internet and but calls to set repos via api
+    // use whatever they set
+    String buildDir = tmpFolder.getRoot().getAbsolutePath();
+    AmbariMetaInfo ambariMetaInfo = setupTempAmbariMetaInfo(buildDir);
+    // The current stack already has (HDP, 2.1.1, redhat6)
+
+    // Updating the baseUrl
+    String newBaseUrl = "http://myprivate-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0";
+    ambariMetaInfo.updateRepoBaseURL(STACK_NAME_HDP, "2.1.1", "redhat6",
+        STACK_NAME_HDP + "-2.1.1", newBaseUrl);
+    RepositoryInfo repoInfo = ambariMetaInfo.getRepository(STACK_NAME_HDP, "2.1.1", "redhat6",
+        STACK_NAME_HDP + "-2.1.1");
+    assertEquals(newBaseUrl, repoInfo.getBaseUrl());
+    String prevBaseUrl = repoInfo.getDefaultBaseUrl();
+    ambariMetaInfo.init();
+
+    List<RepositoryInfo> redhat6Repo = ambariMetaInfo.getRepositories(
+        STACK_NAME_HDP, "2.1.1", "redhat6");
+    assertNotNull(redhat6Repo);
+    for (RepositoryInfo ri : redhat6Repo) {
+      if (STACK_NAME_HDP.equals(ri.getRepoName())) {
+        assertEquals(newBaseUrl, ri.getBaseUrl());
+        // defaultBaseUrl and baseUrl should not be same, since it is updated.
+        assertFalse(ri.getBaseUrl().equals(ri.getDefaultBaseUrl()));
+      }
+    }
+
+    // Reset the database with the original baseUrl
+    ambariMetaInfo.updateRepoBaseURL(STACK_NAME_HDP, "2.1.1", "redhat6",
+        STACK_NAME_HDP + "-2.1.1", prevBaseUrl);
+  }
+
+  @Test
+  public void testGetRepositoryNoInternetUpdatedBaseUrl() throws Exception {
+    // Scenario: user has no internet and but calls to set repos via api
+    // use whatever they set
+    String buildDir = tmpFolder.getRoot().getAbsolutePath();
+    AmbariMetaInfo ambariMetaInfo = setupTempAmbariMetaInfo(buildDir);
+    // The current stack already has (HDP, 2.1.1, redhat6).
+
+    // Deleting the json file referenced by the latestBaseUrl to simulate No
+    // Internet.
+    File latestUrlFile = new File(buildDir,
+        "ambari-metaInfo/HDP/2.1.1/repos/hdp.json");
+    FileUtils.deleteQuietly(latestUrlFile);
+    assertTrue(!latestUrlFile.exists());
+
+    // Update baseUrl
+    String newBaseUrl = "http://myprivate-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0";
+    ambariMetaInfo.updateRepoBaseURL("HDP", "2.1.1", "redhat6", "HDP-2.1.1",
+        newBaseUrl);
+    RepositoryInfo repoInfo = ambariMetaInfo.getRepository(STACK_NAME_HDP, "2.1.1", "redhat6",
+        STACK_NAME_HDP + "-2.1.1");
+    assertEquals(newBaseUrl, repoInfo.getBaseUrl());
+    String prevBaseUrl = repoInfo.getDefaultBaseUrl();
+    ambariMetaInfo.init();
+
+    List<RepositoryInfo> redhat6Repo = ambariMetaInfo.getRepositories(
+        STACK_NAME_HDP, "2.1.1", "redhat6");
+    assertNotNull(redhat6Repo);
+    for (RepositoryInfo ri : redhat6Repo) {
+      if (STACK_NAME_HDP.equals(ri.getRepoName())) {
+        // baseUrl should point to the updated baseUrl
+        assertEquals(newBaseUrl, ri.getBaseUrl());
+        assertFalse(ri.getDefaultBaseUrl().equals(ri.getBaseUrl()));
+      }
+    }
+
+    // Reset the database with the original baseUrl
+    ambariMetaInfo.updateRepoBaseURL(STACK_NAME_HDP, "2.1.1", "redhat6",
+        STACK_NAME_HDP + "-2.1.1", prevBaseUrl);
+  }
+
   @Test
   public void isSupportedStack() throws AmbariException {
     boolean supportedStack = metaInfo.isSupportedStack(STACK_NAME_HDP,
@@ -1433,4 +1560,16 @@ public class AmbariMetaInfoTest {
     Assert.assertTrue(set.size() > 0);
 
   }
+
+  private AmbariMetaInfo setupTempAmbariMetaInfo(String buildDir)
+      throws Exception {
+    File stackRootTmp = new File(buildDir + "/ambari-metaInfo");
+    File stackRoot = new File("src/test/resources/stacks");
+    stackRootTmp.mkdir();
+    FileUtils.copyDirectory(stackRoot, stackRootTmp);
+    AmbariMetaInfo ambariMetaInfo = new AmbariMetaInfo(stackRootTmp, new File(
+        "target/version"));
+    injector.injectMembers(ambariMetaInfo);
+    return ambariMetaInfo;
+  }
 }