AMBARI-6242. Error during ambari-server upgradestack command (aonishuk)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java

@@ -24,6 +24,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import javax.ws.rs.core.UriInfo;
+
 import org.apache.ambari.server.controller.ControllerModule;
 import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
@@ -37,14 +39,10 @@ import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
 import com.google.inject.persist.Transactional;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import org.eclipse.jetty.http.HttpStatus;
 
 public class StackUpgradeHelper {
-
-  private static final Logger LOG = LoggerFactory.getLogger(StackUpgradeHelper.class);
+  private static final Logger LOG = LoggerFactory.getLogger
+    (StackUpgradeHelper.class);
 
   private static final String STACK_ID_UPDATE_ACTION = "updateStackId";
   private static final String METAINFO_UPDATE_ACTION = "updateMetaInfo";
@@ -69,7 +67,6 @@ public class StackUpgradeHelper {
 
   /**
    * Add key value to the metainfo table.
-   *
    * @param data
    * @throws SQLException
    */
@@ -94,7 +91,6 @@ public class StackUpgradeHelper {
 
   /**
    * Change the stack id in the Ambari DB.
-   *
    * @param stackInfo
    * @throws SQLException
    */
@@ -102,75 +98,44 @@ public class StackUpgradeHelper {
     if (stackInfo == null || stackInfo.isEmpty()) {
       throw new IllegalArgumentException("Empty stack id. " + stackInfo);
     }
-
+    
     String repoUrl = stackInfo.remove("repo_url");
     String repoUrlOs = stackInfo.remove("repo_url_os");
-
+    
     Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator();
     Map.Entry<String, String> stackEntry = stackIdEntry.next();
 
     String stackName = stackEntry.getKey();
     String stackVersion = stackEntry.getValue();
 
-    LOG.info("Updating stack id, stackName = " + stackName + ", "
-            + "stackVersion = " + stackVersion);
-    
-    if (null != repoUrl && !repoUrl.isEmpty() && repoUrl.startsWith("http")) {
-      if (!repoUrl.trim().endsWith("/")) repoUrl = repoUrl + "/";
-      if (checkURL(repoUrl + "repodata/repomd.xml", 2000)) {
-        stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);
-      }
-    } else {
-      throw (new Exception("Base repo URL not found"));
-    }
+    LOG.info("Updating stack id, stackName = " + stackName + ", " +
+      "stackVersion = "+ stackVersion);
 
     stackUpgradeUtil.updateStackDetails(stackName, stackVersion);
-
-    dbAccessor.updateTable("hostcomponentstate", "current_state",
-            "INSTALLED", "where current_state = 'UPGRADING'");
-  }
-
-  public boolean checkURL(String url, int timeout) throws Exception {
-    int responseCode = 0;
-    try {
-      HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
-      connection.setConnectTimeout(timeout);
-      connection.setReadTimeout(timeout);
-      connection.setRequestMethod("HEAD");
-      responseCode = connection.getResponseCode();
-      if (responseCode != HttpStatus.OK_200) {
-        throw (new Exception("Invalid repository base URL" +
-                ", Responce: "+ HttpStatus.getMessage(responseCode) +
-                ", during check URL: " + url));        
-      } else {
-        return true;
-      } 
-    } catch (IOException exception) {
-      throw (new Exception(exception.getClass().getSimpleName() +
-              ", Responce: "+ HttpStatus.getMessage(responseCode) +
-              ", during check URL: " + url));
+    
+    if (null != repoUrl) {
+      stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);  
     }
+
+    dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'");
   }
 
   private List<String> getValidActions() {
-    return new ArrayList<String>() {
-      {
-        add(STACK_ID_UPDATE_ACTION);
-        add(METAINFO_UPDATE_ACTION);
-      }
-    };
+    return new ArrayList<String>() {{
+      add(STACK_ID_UPDATE_ACTION);
+      add(METAINFO_UPDATE_ACTION);
+    }};
   }
 
   /**
    * Support changes need to support upgrade of Stack
-   *
    * @param args Simple key value json map
    */
   public static void main(String[] args) {
     try {
       if (args.length < 2) {
-        throw new InputMismatchException("Need to provide action, "
-                + "stack name and stack version.");
+        throw new InputMismatchException("Need to provide action, " +
+          "stack name and stack version.");
       }
 
       String action = args[0];
@@ -181,16 +146,17 @@ public class StackUpgradeHelper {
       Gson gson = injector.getInstance(Gson.class);
 
       if (!stackUpgradeHelper.getValidActions().contains(action)) {
-        throw new IllegalArgumentException("Unsupported action. Allowed "
-                + "actions: " + stackUpgradeHelper.getValidActions());
+        throw new IllegalArgumentException("Unsupported action. Allowed " +
+          "actions: " + stackUpgradeHelper.getValidActions());
       }
 
+      
       stackUpgradeHelper.startPersistenceService();
       Map values = gson.fromJson(valueMap, Map.class);
 
       if (action.equals(STACK_ID_UPDATE_ACTION)) {
         stackUpgradeHelper.updateStackVersion(values);
-
+        
       } else if (action.equals(METAINFO_UPDATE_ACTION)) {
 
         stackUpgradeHelper.updateMetaInfo(values);
@@ -199,8 +165,8 @@ public class StackUpgradeHelper {
       stackUpgradeHelper.stopPersistenceService();
 
     } catch (Throwable t) {
-      System.err.println("Caught exception on upgrade. Exiting..." + t.getMessage());
-      System.exit(2);
+      LOG.error("Caught exception on upgrade. Exiting...", t);
+      System.exit(1);
     }
   }
 }

ambari-server/src/main/python/ambari-server.py

@@ -2639,10 +2639,13 @@ def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None):
   check_database_name_property()
 
   stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
-  retcode, stdout, stderr = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
+  retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
 
   if not retcode == 0:
-    raise FatalException(retcode, 'Error executing stack upgrade. ' + stderr)
+    raise FatalException(retcode, 'Stack upgrade failed.')
+  
+  return retcode
+
 
 
 def load_stack_values(version, filename):
@@ -2754,9 +2757,10 @@ def run_schema_upgrade():
 def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
   jdk_path = find_jdk()
   if jdk_path is None:
-    return 1, "", "No JDK found, please run the \"setup\" \
-                    command to install a JDK automatically or install any \
-                    JDK manually to " + JDK_INSTALL_DIR
+    print_error_msg("No JDK found, please run the \"setup\" "
+                    "command to install a JDK automatically or install any "
+                    "JDK manually to " + JDK_INSTALL_DIR)
+    return 1
   stackId = {}
   stackId[stackName] = stackVersion
   if repo_url is not None:
@@ -2767,7 +2771,11 @@ def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
   command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(),
                                              "updateStackId",
                                             "'" + json.dumps(stackId) + "'")
-  return run_os_command(command)
+  (retcode, stdout, stderr) = run_os_command(command)
+  print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode))
+  if retcode > 0:
+    print_error_msg("Error executing stack upgrade, please check the server logs.")
+  return retcode
 
 
 def run_metainfo_upgrade(keyValueMap=None):

ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeHelperTest.java

@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.upgrade;
-
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.easymock.PowerMock;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.*;
-import static org.powermock.api.easymock.PowerMock.replayAll;
-
-/**
- *
- * @author root
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({StackUpgradeHelper.class})
-public class StackUpgradeHelperTest {
-  
-  public StackUpgradeHelperTest() {
-  }
-
-  @Test
-  public void testUpdateStackVersion()  throws Exception {
-    System.out.println("updateStackVersion");
-    String repoUrl="http://foo.bar";
-    String fullUrl=repoUrl+"/repodata/repomd.xml";
-    URL url = PowerMock.createMockAndExpectNew(URL.class, fullUrl);
-    HttpURLConnection urlConnectionMock = PowerMock.createNiceMock(HttpURLConnection.class);
-    expect(url.openConnection()).andReturn(urlConnectionMock);
-    expect(urlConnectionMock.getResponseCode()).andThrow(new UnknownHostException(fullUrl));
-    PowerMock.replayAll();
-    Map<String, String> stackInfo = new HashMap<String, String>();
-    stackInfo.put("repo_url", repoUrl);
-    stackInfo.put("repo_url_os", "centos6");
-    stackInfo.put("HDP", "1.3.0");
-    StackUpgradeHelper instance = new StackUpgradeHelper();
-    try {
-      instance.updateStackVersion(stackInfo);
-    } catch (Exception ex) {
-      assertEquals("UnknownHostException, Responce: 0, "
-       + "during check URL: " + fullUrl, ex.getMessage());
-    }
-  }
-
-  
-}

ambari-server/src/test/python/TestAmbariServer.py

@@ -1091,7 +1091,7 @@ class TestAmbariServer(TestCase):
       # Expected
       self.assertTrue("JCE Policy path" in fe.reason)
       pass
-    os_path_exists_mock.reset_mock()
+    os_path_exists_mock.reset()
 
     # Case when JCE is a directory
     os_path_exists_mock.return_value = True
@@ -1103,7 +1103,7 @@ class TestAmbariServer(TestCase):
       # Expected
       self.assertTrue("JCE Policy path is a directory" in fe.reason)
       pass
-    os_path_isdir_mock.reset_mock()
+    os_path_isdir_mock.reset()
 
     os_path_isdir_mock.return_value = False
     os_path_join_mock.return_value = \
@@ -2847,12 +2847,11 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     self.assertTrue(removeMock.called)
 
 
-  @patch.object(ambari_server, "run_in_shell")
   @patch.object(ambari_server, "is_root")
   @patch.object(ambari_server, "check_database_name_property")
   @patch.object(ambari_server, "run_stack_upgrade")
   def test_upgrade_stack(self, run_stack_upgrade_mock,
-                         check_database_name_property_mock, is_root_mock, run_in_shell_mock):
+                         check_database_name_property_mock, is_root_mock):
     args = MagicMock()
     args.persistence_type = "local"
 
@@ -2867,43 +2866,12 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
       pass
 
     # Testing calls under root
-    run_in_shell_mock.return_value = 0, "existed!", ""
     is_root_mock.return_value = True
-    run_stack_upgrade_mock.return_value = 0, "", ""
+    run_stack_upgrade_mock.return_value = 0
     ambari_server.upgrade_stack(args, 'HDP-2.0')
 
     self.assertTrue(run_stack_upgrade_mock.called)
     run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None)
-    run_stack_upgrade_mock.reset_mock()
-
-    # Testing calls if desired stack repo not existed
-    run_in_shell_mock.return_value = 1, "", ""
-    is_root_mock.return_value = True
-
-    run_stack_upgrade_mock.reset_mock()
-
-    # Testing calls if desired stack repo not existed but base URL is not empty
-    run_in_shell_mock.return_value = 0, "existed!", ""
-    is_root_mock.return_value = True
-    run_stack_upgrade_mock.return_value = 0, "", ""
-    ambari_server.upgrade_stack(args, 'HDP-2.0', "URL")
-
-    self.assertTrue(run_stack_upgrade_mock.called)
-    run_stack_upgrade_mock.assert_called_with("HDP", "2.0", "URL", None)
-    run_stack_upgrade_mock.reset_mock()
-
-    # Testing calls if upgrade stack return non zero retcode
-    run_in_shell_mock.return_value = 0, "", ""
-    is_root_mock.return_value = True
-    run_stack_upgrade_mock.return_value = 2, "", ""
-    try:
-      ambari_server.upgrade_stack(args, 'HDP-2.0', "URL")
-      self.fail("Should throw exception")
-    except FatalException as fe:
-      # Expected
-      self.assertTrue("Error executing stack upgrade." in fe.reason)
-      pass
-
 
   @patch.object(ambari_server, 'get_conf_dir')
   @patch.object(ambari_server, 'get_ambari_classpath')
@@ -3834,7 +3802,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     copy_files_mock.assert_called_with(drivers_list, resources_dir)
 
     # Non-Silent option, no drivers at first ask, present drivers after that
-    find_jdbc_driver_mock.reset_mock()
+    find_jdbc_driver_mock.reset()
     find_jdbc_driver_mock.side_effect = [-1, -1]
 
     rcode = ambari_server.check_jdbc_drivers(args)