AMBARI-4894. Allow upgrade stack to take URL as a parameter (ncole)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java

@@ -17,25 +17,28 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import com.google.gson.Gson;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.PersistService;
-import com.google.inject.persist.Transactional;
-import org.apache.ambari.server.controller.ControllerModule;
-import org.apache.ambari.server.orm.DBAccessor;
-import org.apache.ambari.server.orm.dao.MetainfoDAO;
-import org.apache.ambari.server.orm.entities.MetainfoEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.InputMismatchException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.ambari.server.controller.ControllerModule;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+import com.google.inject.persist.Transactional;
 
 public class StackUpgradeHelper {
   private static final Logger LOG = LoggerFactory.getLogger
@@ -88,14 +91,18 @@ public class StackUpgradeHelper {
 
   /**
    * Change the stack id in the Ambari DB.
-   * @param stackId
+   * @param stackInfo
    * @throws SQLException
    */
-  private void updateStackVersion(Map<String, String> stackId) throws SQLException {
-    if (stackId == null || stackId.isEmpty()) {
-      throw new IllegalArgumentException("Empty stack id. " + stackId);
+  public void updateStackVersion(Map<String, String> stackInfo) throws Exception {
+    if (stackInfo == null || stackInfo.isEmpty()) {
+      throw new IllegalArgumentException("Empty stack id. " + stackInfo);
     }
-    Iterator<Map.Entry<String, String>> stackIdEntry = stackId.entrySet().iterator();
+    
+    String repoUrl = stackInfo.remove("repo_url");
+    String repoUrlOs = stackInfo.remove("repo_url_os");
+    
+    Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator();
     Map.Entry<String, String> stackEntry = stackIdEntry.next();
 
     String stackName = stackEntry.getKey();
@@ -105,6 +112,10 @@ public class StackUpgradeHelper {
       "stackVersion = "+ stackVersion);
 
     stackUpgradeUtil.updateStackDetails(stackName, stackVersion);
+    
+    if (null != repoUrl) {
+      stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);  
+    }
 
     dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'");
   }
@@ -139,11 +150,13 @@ public class StackUpgradeHelper {
           "actions: " + stackUpgradeHelper.getValidActions());
       }
 
+      
       stackUpgradeHelper.startPersistenceService();
       Map values = gson.fromJson(valueMap, Map.class);
 
       if (action.equals(STACK_ID_UPDATE_ACTION)) {
         stackUpgradeHelper.updateStackVersion(values);
+        
       } else if (action.equals(METAINFO_UPDATE_ACTION)) {
 
         stackUpgradeHelper.updateMetaInfo(values);

ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java

@@ -17,10 +17,14 @@
  */
 package org.apache.ambari.server.upgrade;
 
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.Transactional;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Scanner;
+import java.util.Set;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.orm.dao.ClusterDAO;
 import org.apache.ambari.server.orm.dao.ClusterStateDAO;
 import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO;
@@ -33,9 +37,13 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
 import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
 import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
+import org.apache.ambari.server.state.OperatingSystemInfo;
 import org.apache.ambari.server.state.StackId;
-import java.util.ArrayList;
-import java.util.List;
+
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.Transactional;
 
 public class StackUpgradeUtil {
   @Inject
@@ -139,4 +147,43 @@ public class StackUpgradeUtil {
 
 
   }
+
+  /**
+   * @param stackName
+   * @param stackVersion
+   * @param localRepo
+   */
+  public void updateLocalRepo(String stackName, String stackVersion,
+      String repoUrl, String repoUrlOs) throws Exception {
+
+    if (null == repoUrl ||
+        repoUrl.isEmpty() ||
+        !repoUrl.startsWith("http"))
+      return;
+    
+    String server = repoUrl;
+    
+    String[] oses = new String[0]; 
+    
+    if (null != repoUrlOs) {
+      oses = repoUrlOs.split(",");
+    }
+    
+    AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class);
+    
+    if (0 == oses.length) {
+      // do them all
+      for (OperatingSystemInfo osi : ami.getOperatingSystems(stackName, stackVersion)) {
+        ami.updateRepoBaseURL(stackName, stackVersion, osi.getOsType(),
+            stackName + "-" + stackVersion, server);
+      }
+      
+    } else {
+      for (String os : oses) {
+        ami.updateRepoBaseURL(stackName, stackVersion, os,
+            stackName + "-" + stackVersion, server);
+      }
+    }
+  }
+
 }

ambari-server/src/main/python/ambari-server.py

@@ -2487,7 +2487,7 @@ def stop(args):
 ### Stack upgrade ###
 
 
-def upgrade_stack(args, stack_id):
+def upgrade_stack(args, stack_id, repo_url = None, repo_url_os = None):
   if not is_root():
     err = 'Ambari-server upgradestack should be run with ' \
           'root-level privileges'
@@ -2495,7 +2495,7 @@ def upgrade_stack(args, stack_id):
   check_database_name_property()
 
   stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
-  retcode = run_stack_upgrade(stack_name, stack_version)
+  retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
 
   if not retcode == 0:
     raise FatalException(retcode, 'Stack upgrade failed.')
@@ -2606,7 +2606,7 @@ def run_schema_upgrade(version):
     print_error_msg("Error executing schema upgrade, please check the server logs.")
   return retcode
 
-def run_stack_upgrade(stackName, stackVersion):
+def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
   jdk_path = find_jdk()
   if jdk_path is None:
     print_error_msg("No JDK found, please run the \"setup\" "
@@ -2615,6 +2615,10 @@ def run_stack_upgrade(stackName, stackVersion):
     return 1
   stackId = {}
   stackId[stackName] = stackVersion
+  if repo_url is not None:
+    stackId['repo_url'] = repo_url
+  if repo_url_os is not None:
+    stackId['repo_url_os'] = repo_url_os
 
   command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(),
                                              "updateStackId",
@@ -4070,7 +4074,15 @@ def main():
       upgrade(options)
     elif action == UPGRADE_STACK_ACTION:
       stack_id = args[1]
-      upgrade_stack(options, stack_id)
+      repo_url = None
+      repo_url_os = None
+
+      if len(args) > 2:
+        repo_url = args[2]
+      if len(args) > 3:
+        repo_url_os = args[3]
+
+      upgrade_stack(options, stack_id, repo_url, repo_url_os)
     elif action == LDAP_SETUP_ACTION:
       setup_ldap()
     elif action == SETUP_SECURITY_ACTION:

ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java

@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.GuiceJpaInitializer;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+
+/**
+ * Tests the StackUpgradeHelper
+ */
+public class StackUpgradeUtilTest {
+
+  private Injector injector;
+  
+  @Before
+  public void setup() throws Exception {
+    injector = Guice.createInjector(new InMemoryDefaultTestModule());
+    injector.getInstance(GuiceJpaInitializer.class);
+  }
+  
+  @After
+  public void teardown() throws Exception {
+    injector.getInstance(PersistService.class).stop();
+  }
+  
+  private void reset(String stackName, String stackVersion) throws Exception {
+    AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class);
+    
+    for (Entry<String, List<RepositoryInfo>> entry : ami.getRepository(stackName, stackVersion).entrySet()) {
+      for (RepositoryInfo ri : entry.getValue()) {
+        if (-1 == ri.getRepoId().indexOf("epel")) {
+          ami.updateRepoBaseURL(stackName, stackVersion,
+              ri.getOsType(), ri.getRepoId(), ri.getDefaultBaseUrl());
+        }
+      }
+    }
+    
+  }
+  
+  @Test
+  public void testUpgradeStack() throws Exception {
+    StackUpgradeUtil stackUpgradeUtil = injector.getInstance(StackUpgradeUtil.class);
+    
+    String stackName = "HDP";
+    String stackVersion = "1.3.0";
+    String localRepoUrl = "http://foo.bar";
+    
+    // check updating all
+    stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, null);
+    
+    MetainfoDAO dao = injector.getInstance(MetainfoDAO.class);
+    
+    Collection<MetainfoEntity> entities = dao.findAll();
+    Assert.assertTrue(entities.size() > 0);
+    
+    for (MetainfoEntity entity : entities) {
+      Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/"));
+      Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+    }
+    
+    reset (stackName, stackVersion);
+    entities = dao.findAll();
+    Assert.assertTrue(0 == entities.size());
+    
+    // check updating only centos6
+    stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6");
+
+    entities = dao.findAll();
+    for (MetainfoEntity entity : entities) {
+      Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6"));
+      Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+    }
+
+    reset (stackName, stackVersion);
+    entities = dao.findAll();
+    Assert.assertTrue(0 == entities.size());
+    
+    // check updating only centos6 and centos5
+    stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6,centos5");
+
+    entities = dao.findAll();
+    for (MetainfoEntity entity : entities) {
+      Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6") ||
+          entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos5"));
+      Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+    }
+    
+    
+  }
+  
+  
+}

ambari-server/src/test/python/TestAmbariServer.py

@@ -2740,7 +2740,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     ambari_server.upgrade_stack(args, 'HDP-2.0')
 
     self.assertTrue(run_stack_upgrade_mock.called)
-    run_stack_upgrade_mock.assert_called_with("HDP", "2.0")
+    run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None)
 
   @patch.object(ambari_server, 'get_conf_dir')
   @patch.object(ambari_server, 'get_ambari_classpath')
@@ -2754,7 +2754,51 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     get_conf_dir_mock.return_value = '/etc/conf'
     stackIdMap = {'HDP' : '2.0'}
 
-    ambari_server.run_stack_upgrade('HDP', '2.0')
+    ambari_server.run_stack_upgrade('HDP', '2.0', None, None)
+
+    self.assertTrue(jdk_path_mock.called)
+    self.assertTrue(get_ambari_classpath_mock.called)
+    self.assertTrue(get_conf_dir_mock.called)
+    self.assertTrue(run_os_command_mock.called)
+    run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 '
+                                           'org.apache.ambari.server.upgrade.StackUpgradeHelper '
+        'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1')
+
+  @patch.object(ambari_server, 'get_conf_dir')
+  @patch.object(ambari_server, 'get_ambari_classpath')
+  @patch.object(ambari_server, 'run_os_command')
+  @patch.object(ambari_server, 'find_jdk')
+  def test_run_stack_upgrade_with_url(self, jdk_path_mock, run_os_command_mock,
+                             get_ambari_classpath_mock, get_conf_dir_mock):
+    jdk_path_mock.return_value = "/usr/lib/java"
+    run_os_command_mock.return_value = (0, None, None)
+    get_ambari_classpath_mock.return_value = 'test:path12'
+    get_conf_dir_mock.return_value = '/etc/conf'
+    stackIdMap = {'HDP' : '2.0', 'repo_url' : 'http://test.com'}
+
+    ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', None)
+
+    self.assertTrue(jdk_path_mock.called)
+    self.assertTrue(get_ambari_classpath_mock.called)
+    self.assertTrue(get_conf_dir_mock.called)
+    self.assertTrue(run_os_command_mock.called)
+    run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 '
+                                           'org.apache.ambari.server.upgrade.StackUpgradeHelper '
+        'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1')
+
+  @patch.object(ambari_server, 'get_conf_dir')
+  @patch.object(ambari_server, 'get_ambari_classpath')
+  @patch.object(ambari_server, 'run_os_command')
+  @patch.object(ambari_server, 'find_jdk')
+  def test_run_stack_upgrade_with_url_os(self, jdk_path_mock, run_os_command_mock,
+                             get_ambari_classpath_mock, get_conf_dir_mock):
+    jdk_path_mock.return_value = "/usr/lib/java"
+    run_os_command_mock.return_value = (0, None, None)
+    get_ambari_classpath_mock.return_value = 'test:path12'
+    get_conf_dir_mock.return_value = '/etc/conf'
+    stackIdMap = {'HDP' : '2.0', 'repo_url': 'http://test.com', 'repo_url_os': 'centos5,centos6'}
+
+    ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', 'centos5,centos6')
 
     self.assertTrue(jdk_path_mock.called)
     self.assertTrue(get_ambari_classpath_mock.called)