AMBARI-6008. Incorrect behaviour when trying to execute upgradestack to nonexistent HDP version (aonishuk)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java

@@ -24,8 +24,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import javax.ws.rs.core.UriInfo;
-
 import org.apache.ambari.server.controller.ControllerModule;
 import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
@@ -39,10 +37,14 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
 import com.google.inject.persist.Transactional;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import org.eclipse.jetty.http.HttpStatus;
 
 public class StackUpgradeHelper {
-  private static final Logger LOG = LoggerFactory.getLogger
-    (StackUpgradeHelper.class);
+
+  private static final Logger LOG = LoggerFactory.getLogger(StackUpgradeHelper.class);
 
   private static final String STACK_ID_UPDATE_ACTION = "updateStackId";
   private static final String METAINFO_UPDATE_ACTION = "updateMetaInfo";
@@ -67,6 +69,7 @@ public class StackUpgradeHelper {
 
   /**
    * Add key value to the metainfo table.
+   *
    * @param data
    * @throws SQLException
    */
@@ -91,6 +94,7 @@ public class StackUpgradeHelper {
 
   /**
    * Change the stack id in the Ambari DB.
+   *
    * @param stackInfo
    * @throws SQLException
    */
@@ -98,44 +102,75 @@ public class StackUpgradeHelper {
     if (stackInfo == null || stackInfo.isEmpty()) {
       throw new IllegalArgumentException("Empty stack id. " + stackInfo);
     }
-    
+
     String repoUrl = stackInfo.remove("repo_url");
     String repoUrlOs = stackInfo.remove("repo_url_os");
-    
+
     Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator();
     Map.Entry<String, String> stackEntry = stackIdEntry.next();
 
     String stackName = stackEntry.getKey();
     String stackVersion = stackEntry.getValue();
 
-    LOG.info("Updating stack id, stackName = " + stackName + ", " +
-      "stackVersion = "+ stackVersion);
-
-    stackUpgradeUtil.updateStackDetails(stackName, stackVersion);
+    LOG.info("Updating stack id, stackName = " + stackName + ", "
+            + "stackVersion = " + stackVersion);
     
-    if (null != repoUrl) {
-      stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);  
+    if (null != repoUrl && !repoUrl.isEmpty() && repoUrl.startsWith("http")) {
+      if (!repoUrl.trim().endsWith("/")) repoUrl = repoUrl + "/";
+      if (checkURL(repoUrl + "repodata/repomd.xml", 2000)) {
+        stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);
+      }
+    } else {
+      throw (new Exception("Base repo URL not found"));
     }
 
-    dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'");
+    stackUpgradeUtil.updateStackDetails(stackName, stackVersion);
+
+    dbAccessor.updateTable("hostcomponentstate", "current_state",
+            "INSTALLED", "where current_state = 'UPGRADING'");
+  }
+
+  public boolean checkURL(String url, int timeout) throws Exception {
+    int responseCode = 0;
+    try {
+      HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
+      connection.setConnectTimeout(timeout);
+      connection.setReadTimeout(timeout);
+      connection.setRequestMethod("HEAD");
+      responseCode = connection.getResponseCode();
+      if (responseCode != HttpStatus.OK_200) {
+        throw (new Exception("Invalid repository base URL" +
+                ", Responce: "+ HttpStatus.getMessage(responseCode) +
+                ", during check URL: " + url));        
+      } else {
+        return true;
+      } 
+    } catch (IOException exception) {
+      throw (new Exception(exception.getClass().getSimpleName() +
+              ", Responce: "+ HttpStatus.getMessage(responseCode) +
+              ", during check URL: " + url));
+    }
   }
 
   private List<String> getValidActions() {
-    return new ArrayList<String>() {{
-      add(STACK_ID_UPDATE_ACTION);
-      add(METAINFO_UPDATE_ACTION);
-    }};
+    return new ArrayList<String>() {
+      {
+        add(STACK_ID_UPDATE_ACTION);
+        add(METAINFO_UPDATE_ACTION);
+      }
+    };
   }
 
   /**
    * Support changes need to support upgrade of Stack
+   *
    * @param args Simple key value json map
    */
   public static void main(String[] args) {
     try {
       if (args.length < 2) {
-        throw new InputMismatchException("Need to provide action, " +
-          "stack name and stack version.");
+        throw new InputMismatchException("Need to provide action, "
+                + "stack name and stack version.");
       }
 
       String action = args[0];
@@ -146,17 +181,16 @@ public class StackUpgradeHelper {
       Gson gson = injector.getInstance(Gson.class);
 
       if (!stackUpgradeHelper.getValidActions().contains(action)) {
-        throw new IllegalArgumentException("Unsupported action. Allowed " +
-          "actions: " + stackUpgradeHelper.getValidActions());
+        throw new IllegalArgumentException("Unsupported action. Allowed "
+                + "actions: " + stackUpgradeHelper.getValidActions());
       }
 
-      
       stackUpgradeHelper.startPersistenceService();
       Map values = gson.fromJson(valueMap, Map.class);
 
       if (action.equals(STACK_ID_UPDATE_ACTION)) {
         stackUpgradeHelper.updateStackVersion(values);
-        
+
       } else if (action.equals(METAINFO_UPDATE_ACTION)) {
 
         stackUpgradeHelper.updateMetaInfo(values);
@@ -165,8 +199,8 @@ public class StackUpgradeHelper {
       stackUpgradeHelper.stopPersistenceService();
 
     } catch (Throwable t) {
-      LOG.error("Caught exception on upgrade. Exiting...", t);
-      System.exit(1);
+      System.err.println("Caught exception on upgrade. Exiting..." + t.getMessage());
+      System.exit(2);
     }
   }
 }

ambari-server/src/main/python/ambari-server.py

@@ -2605,13 +2605,26 @@ def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None):
     raise FatalException(4, err)
   check_database_name_property()
 
+  local_repo_check_commamd = ""
+  if OS_FAMILY == OSConst.DEBIAN_FAMILY:
+    raise FatalException(3, '"upgradestack" command not supported yet for Debian OS\'es family.')
+  elif OS_FAMILY == OSConst.REDHAT_FAMILY:
+    local_repo_check_commamd = 'yum repolist | grep "{0} "'
+  elif OS_FAMILY == OSConst.SUSE_FAMILY:
+    local_repo_check_commamd = 'zypper repos | grep "{0} "'
+  
+  command = local_repo_check_commamd.format(stack_id)
+  (retcode, stdout, stderr) = run_in_shell(command)
+  
+  if not retcode == 0 and repo_url is None:
+    raise FatalException(retcode, 'Repository for ' + stack_id + " is not existed")
+
   stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
-  retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
+  retcode, stdout, stderr = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
 
   if not retcode == 0:
-    raise FatalException(retcode, 'Stack upgrade failed.')
+    raise FatalException(retcode, 'Error executing stack upgrade. ' + stderr)
 
-  return retcode
 
 
 def load_stack_values(version, filename):
@@ -2723,10 +2736,9 @@ def run_schema_upgrade():
 def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
   jdk_path = find_jdk()
   if jdk_path is None:
-    print_error_msg("No JDK found, please run the \"setup\" "
-                    "command to install a JDK automatically or install any "
-                    "JDK manually to " + JDK_INSTALL_DIR)
-    return 1
+    return 1, "", "No JDK found, please run the \"setup\" \
+                    command to install a JDK automatically or install any \
+                    JDK manually to " + JDK_INSTALL_DIR
   stackId = {}
   stackId[stackName] = stackVersion
   if repo_url is not None:
@@ -2737,11 +2749,7 @@ def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
   command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(),
                                              "updateStackId",
                                             "'" + json.dumps(stackId) + "'")
-  (retcode, stdout, stderr) = run_os_command(command)
-  print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
-  if retcode > 0:
-    print_error_msg("Error executing stack upgrade, please check the server logs.")
-  return retcode
+  return run_os_command(command)
 
 
 def run_metainfo_upgrade(keyValueMap=None):

ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author root
+ */
+public class StackUpgradeHelperTest {
+  
+  public StackUpgradeHelperTest() {
+  }
+
+  @Test
+  public void testUpdateStackVersion() {
+    System.out.println("updateStackVersion");
+    Map<String, String> stackInfo = new HashMap<String, String>();
+    stackInfo.put("repo_url", "http://foo.bar");
+    stackInfo.put("repo_url_os", "centos6");
+    stackInfo.put("HDP", "1.3.0");
+    StackUpgradeHelper instance = new StackUpgradeHelper();
+    try {
+      instance.updateStackVersion(stackInfo);
+    } catch (Exception ex) {
+      assertEquals("UnknownHostException, Responce: 0, "
+       + "during check URL: http://foo.bar/repodata/repomd.xml", ex.getMessage());
+    }
+  }
+
+  
+}

ambari-server/src/test/python/TestAmbariServer.py

@@ -1091,7 +1091,7 @@ class TestAmbariServer(TestCase):
       # Expected
       self.assertTrue("JCE Policy path" in fe.reason)
       pass
-    os_path_exists_mock.reset()
+    os_path_exists_mock.reset_mock()
 
     # Case when JCE is a directory
     os_path_exists_mock.return_value = True
@@ -1103,7 +1103,7 @@ class TestAmbariServer(TestCase):
       # Expected
       self.assertTrue("JCE Policy path is a directory" in fe.reason)
       pass
-    os_path_isdir_mock.reset()
+    os_path_isdir_mock.reset_mock()
 
     os_path_isdir_mock.return_value = False
     os_path_join_mock.return_value = \
@@ -2783,11 +2783,12 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     self.assertTrue(removeMock.called)
 
 
+  @patch.object(ambari_server, "run_in_shell")
   @patch.object(ambari_server, "is_root")
   @patch.object(ambari_server, "check_database_name_property")
   @patch.object(ambari_server, "run_stack_upgrade")
   def test_upgrade_stack(self, run_stack_upgrade_mock,
-                         check_database_name_property_mock, is_root_mock):
+                         check_database_name_property_mock, is_root_mock, run_in_shell_mock):
     args = MagicMock()
     args.persistence_type = "local"
 
@@ -2802,12 +2803,50 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
       pass
 
     # Testing calls under root
+    run_in_shell_mock.return_value = 0, "existed!", ""
     is_root_mock.return_value = True
-    run_stack_upgrade_mock.return_value = 0
+    run_stack_upgrade_mock.return_value = 0, "", ""
     ambari_server.upgrade_stack(args, 'HDP-2.0')
 
     self.assertTrue(run_stack_upgrade_mock.called)
     run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None)
+    run_stack_upgrade_mock.reset_mock()
+
+    # Testing calls if desired stack repo not existed
+    run_in_shell_mock.return_value = 1, "", ""
+    is_root_mock.return_value = True
+    try:
+      ambari_server.upgrade_stack(args, 'HDP-2.0')
+      self.fail("Should throw exception")
+    except FatalException as fe:
+      # Expected
+      self.assertTrue("Repository for HDP-2.0 is not existed" in fe.reason)
+      pass
+
+    run_stack_upgrade_mock.reset_mock()
+
+    # Testing calls if desired stack repo not existed but base URL is not empty
+    run_in_shell_mock.return_value = 0, "existed!", ""
+    is_root_mock.return_value = True
+    run_stack_upgrade_mock.return_value = 0, "", ""
+    ambari_server.upgrade_stack(args, 'HDP-2.0', "URL")
+
+    self.assertTrue(run_stack_upgrade_mock.called)
+    run_stack_upgrade_mock.assert_called_with("HDP", "2.0", "URL", None)
+    run_stack_upgrade_mock.reset_mock()
+
+    # Testing calls if upgrade stack return non zero retcode
+    run_in_shell_mock.return_value = 0, "", ""
+    is_root_mock.return_value = True
+    run_stack_upgrade_mock.return_value = 2, "", ""
+    try:
+      ambari_server.upgrade_stack(args, 'HDP-2.0', "URL")
+      self.fail("Should throw exception")
+    except FatalException as fe:
+      # Expected
+      self.assertTrue("Error executing stack upgrade." in fe.reason)
+      pass
+
 
   @patch.object(ambari_server, 'get_conf_dir')
   @patch.object(ambari_server, 'get_ambari_classpath')
@@ -3734,7 +3773,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     copy_files_mock.assert_called_with(drivers_list, resources_dir)
 
     # Non-Silent option, no drivers at first ask, present drivers after that
-    find_jdbc_driver_mock.reset()
+    find_jdbc_driver_mock.reset_mock()
     find_jdbc_driver_mock.side_effect = [-1, -1]
 
     rcode = ambari_server.check_jdbc_drivers(args)