AMBARI-10017. Rolling Upgrade: Expose FinalizeUpgradeAction to API or force transition from UPGRADED to CURRENT for cluster_version via API request. Additional fixes (dlysnichenko)

ambari-server/sbin/ambari-server

@@ -123,6 +123,10 @@ case "$1" in
         echo -e "Syncing with LDAP..."
         $PYTHON /usr/sbin/ambari-server.py $@
         ;;
+  set-current)
+        echo -e "Setting current version..."
+        $PYTHON /usr/sbin/ambari-server.py $@
+        ;;
   setup-security)
         echo -e "Security setup options..."
         $PYTHON /usr/sbin/ambari-server.py $@
@@ -141,7 +145,7 @@ case "$1" in
         ;;
   *)
         echo "Usage: /usr/sbin/ambari-server
-        {start|stop|restart|setup|setup-jce|upgrade|status|upgradestack|setup-ldap|sync-ldap|setup-security|refresh-stack-hash|backup|restore} [options]
+        {start|stop|restart|setup|setup-jce|upgrade|status|upgradestack|setup-ldap|sync-ldap|set-current|setup-security|refresh-stack-hash|backup|restore} [options]
         Use usr/sbin/ambari-server <action> --help to get details on options available.
         Or, simply invoke ambari-server.py --help to print the options."
         exit 1

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java

@@ -499,13 +499,16 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
       CommandReport report = finalizeUpgradeAction.execute(null);
 
       LOG.info("Finalize output:");
-      LOG.info("STDERR: {}", report.getStdErr());
       LOG.info("STDOUT: {}", report.getStdOut());
+      LOG.info("STDERR: {}", report.getStdErr());
 
       if (report.getStatus().equals(HostRoleStatus.COMPLETED.toString())) {
         return getRequestStatus(null);
       } else {
-        throw new SystemException("Finalization failed");
+        String detailedOutput = "Finalization failed. More details: \n" +
+                "STDOUT: " + report.getStdOut() + "\n" +
+                "STDERR: " + report.getStdErr();
+        throw new SystemException(detailedOutput);
       }
     } catch (AmbariException e) {
       throw new SystemException("Can not perform request", e);

ambari-server/src/main/python/ambari-server.py

@@ -34,12 +34,12 @@ from ambari_server.dbConfiguration import DATABASE_NAMES
 from ambari_server.serverConfiguration import configDefaults, get_ambari_properties, PID_NAME
 from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
 from ambari_server.serverSetup import reset, setup, setup_jce_policy
-from ambari_server.serverUpgrade import upgrade, upgrade_stack
+from ambari_server.serverUpgrade import upgrade, upgrade_stack, set_current
 from ambari_server.setupHttps import setup_https
 
 from ambari_server.setupActions import BACKUP_ACTION, LDAP_SETUP_ACTION, LDAP_SYNC_ACTION, PSTART_ACTION, \
   REFRESH_STACK_HASH_ACTION, RESET_ACTION, RESTORE_ACTION, SETUP_ACTION, SETUP_SECURITY_ACTION, START_ACTION, \
-  STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, SETUP_JCE_ACTION
+  STATUS_ACTION, STOP_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, SETUP_JCE_ACTION, SET_CURRENT
 from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas
 from ambari_server.userInput import get_validated_string_input
 
@@ -360,6 +360,8 @@ def init_parser_options(parser):
   parser.add_option('--jdbc-db', default=None, help="Specifies the database type [postgres|mysql|oracle] for the " \
                                                     "JDBC driver specified with the --jdbc-driver option. Used only with --jdbc-driver option.",
                     dest="jdbc_db")
+  parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name")
+  parser.add_option('--version-display-name', default=None, help="Display name of desired repo version", dest="desired_repo_version")
 
 
 @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
@@ -498,6 +500,7 @@ def create_user_action_map(args, options):
         UPGRADE_STACK_ACTION: UserActionPossibleArgs(upgrade_stack, [2, 4], args),
         LDAP_SETUP_ACTION: UserAction(setup_ldap),
         LDAP_SYNC_ACTION: UserAction(sync_ldap, options),
+        SET_CURRENT: UserAction(set_current, options),
         SETUP_SECURITY_ACTION: UserActionRestart(setup_security, options),
         REFRESH_STACK_HASH_ACTION: UserAction(refresh_stack_hash_action),
         BACKUP_ACTION: UserActionPossibleArgs(backup, [1, 2], args),

ambari-server/src/main/python/ambari_server/serverUpgrade.py

@@ -20,10 +20,13 @@ limitations under the License.
 
 import json
 import os
+import sys
 import shutil
+import base64
+import urllib2
 
 from ambari_commons.exceptions import FatalException
-from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg
+from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, get_verbose
 from ambari_commons.os_utils import is_root, run_os_command
 from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers
 from ambari_server.properties import Properties
@@ -35,6 +38,8 @@ from ambari_server.serverConfiguration import configDefaults, \
   SETUP_OR_UPGRADE_MSG
 from ambari_server.setupSecurity import adjust_directory_permissions
 from ambari_server.utils import compare_versions
+from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base
+from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
 
 # constants
 STACK_NAME_VER_SEP = "-"
@@ -323,3 +328,83 @@ def upgrade(args):
   # check if ambari has obsolete LDAP configuration
   if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
     args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
+
+
+#
+# Set current cluster version (run Finalize during manual RU)
+#
+def set_current(options):
+  server_status, pid = is_server_runing()
+  if not server_status:
+    err = 'Ambari Server is not running.'
+    raise FatalException(1, err)
+
+  finalize_options = SetCurrentVersionOptions(options)
+
+  if finalize_options.no_finalize_options_set():
+    err = 'Must specify --cluster-name and --version-display-name. Please invoke ambari-server.py --help to print the options.'
+    raise FatalException(1, err)
+
+  admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: ", default=None,
+                                           pattern=None, description=None,
+                                           is_pass=False, allowEmpty=False)
+  admin_password = get_validated_string_input(prompt="Enter Ambari Admin password: ", default=None,
+                                              pattern=None, description=None,
+                                              is_pass=True, allowEmpty=False)
+
+  properties = get_ambari_properties()
+  if properties == -1:
+    raise FatalException(1, "Failed to read properties file.")
+
+  base_url = get_ambari_server_api_base(properties)
+  url = base_url + "clusters/{0}/stack_versions".format(finalize_options.cluster_name)
+  admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '')
+  request = urllib2.Request(url)
+  request.add_header('Authorization', 'Basic %s' % admin_auth)
+  request.add_header('X-Requested-By', 'ambari')
+
+  data = {
+    "ClusterStackVersions": {
+      "repository_version": finalize_options.desired_repo_version,
+      "state": "CURRENT"
+    }
+  }
+
+  if get_verbose():
+    sys.stdout.write('\nCalling API ' + url + ' : ' + str(data) + '\n')
+
+  request.add_data(json.dumps(data))
+  request.get_method = lambda: 'PUT'
+
+  try:
+    response = urllib2.urlopen(request)
+  except urllib2.HTTPError, e:
+    code = e.getcode()
+    content = e.read()
+    err = 'Error during setting current version. Http status code - {0}. \n {1}'.format(
+      code, content)
+    raise FatalException(1, err)
+  except Exception as e:
+    err = 'Setting current version failed. Error details: %s' % e
+    raise FatalException(1, err)
+
+  sys.stdout.write('\nCurrent version successfully updated to ' + finalize_options.desired_repo_version)
+
+  sys.stdout.write('\n')
+  sys.stdout.flush()
+
+
+class SetCurrentVersionOptions:
+  def __init__(self, options):
+    try:
+      self.cluster_name = options.cluster_name
+    except AttributeError:
+      self.cluster_name = None
+
+    try:
+      self.desired_repo_version = options.desired_repo_version
+    except AttributeError:
+      self.desired_repo_version = None
+
+  def no_finalize_options_set(self):
+    return self.cluster_name is None or self.desired_repo_version is None

ambari-server/src/main/python/ambari_server/serverUtils.py

@@ -25,9 +25,15 @@ from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.os_check import OSConst
 from ambari_commons.os_utils import run_os_command
 from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
-from ambari_server.serverConfiguration import configDefaults, PID_NAME, get_ambari_properties, get_stack_location
+from ambari_server.serverConfiguration import configDefaults, PID_NAME, get_ambari_properties, get_stack_location, \
+  CLIENT_API_PORT, SSL_API, DEFAULT_SSL_API_PORT, SSL_API_PORT
 
 
+# Ambari server API properties
+SERVER_API_HOST = '127.0.0.1'
+SERVER_API_PROTOCOL = 'http'
+SERVER_API_SSL_PROTOCOL = 'https'
+
 @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
 def is_server_runing():
   pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
@@ -90,3 +96,27 @@ def refresh_stack_hash(properties):
     msg = "Can not organize resource files at {0}: {1}".format(
       resources_location, str(ex))
     raise FatalException(-1, msg)
+
+
+#
+# Builds ambari-server API base url
+# Reads server protocol/port from configuration
+# And returns something like
+# http://127.0.0.1:8080:/api/v1/
+#
+def get_ambari_server_api_base(properties):
+  api_protocol = SERVER_API_PROTOCOL
+  api_port = CLIENT_API_PORT
+
+  api_ssl = False
+  api_ssl_prop = properties.get_property(SSL_API)
+  if api_ssl_prop is not None:
+    api_ssl = api_ssl_prop.lower() == "true"
+
+  if api_ssl:
+    api_protocol = SERVER_API_SSL_PROTOCOL
+    api_port = DEFAULT_SSL_API_PORT
+    api_port_prop = properties.get_property(SSL_API_PORT)
+    if api_port_prop is not None:
+      api_port = api_port_prop
+  return '{0}://{1}:{2!s}/api/v1/'.format(api_protocol, SERVER_API_HOST, api_port)

ambari-server/src/main/python/ambari_server/setupActions.py

@@ -31,6 +31,7 @@ STATUS_ACTION = "status"
 SETUP_HTTPS_ACTION = "setup-https"
 LDAP_SETUP_ACTION = "setup-ldap"
 LDAP_SYNC_ACTION = "sync-ldap"
+SET_CURRENT = "set-current"
 SETUP_GANGLIA_HTTPS_ACTION = "setup-ganglia-https"
 ENCRYPT_PASSWORDS_ACTION = "encrypt-passwords"
 SETUP_SECURITY_ACTION = "setup-security"

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -45,7 +45,7 @@ from ambari_server.serverConfiguration import configDefaults, \
   SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
   SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
   SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT
-from ambari_server.serverUtils import is_server_runing
+from ambari_server.serverUtils import is_server_runing, get_ambari_server_api_base
 from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
 from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
 
@@ -59,11 +59,7 @@ REGEX_ANYTHING = ".*"
 
 CLIENT_SECURITY_KEY = "client.security"
 
-# Ambari server API properties
-SERVER_API_HOST = '127.0.0.1'
-SERVER_API_PROTOCOL = 'http'
-SERVER_API_SSL_PROTOCOL = 'https'
-SERVER_API_LDAP_URL = '/api/v1/ldap_sync_events'
+SERVER_API_LDAP_URL = 'ldap_sync_events'
 
 
 def read_master_key(isReset=False):
@@ -225,7 +221,11 @@ def sync_ldap(options):
     err = 'Ambari Server is not running.'
     raise FatalException(1, err)
 
-  ldap_configured = get_ambari_properties().get_property(IS_LDAP_CONFIGURED)
+  properties = get_ambari_properties()
+  if properties == -1:
+    raise FatalException(1, "Failed to read properties file.")
+
+  ldap_configured = properties.get_property(IS_LDAP_CONFIGURED)
   if ldap_configured != 'true':
     err = "LDAP is not configured. Run 'ambari-server setup-ldap' first."
     raise FatalException(1, err)
@@ -244,26 +244,7 @@ def sync_ldap(options):
                                               pattern=None, description=None,
                                               is_pass=True, allowEmpty=False)
 
-  properties = get_ambari_properties()
-  if properties == -1:
-    raise FatalException(1, "Failed to read properties file.")
-
-  api_protocol = SERVER_API_PROTOCOL
-  api_port = CLIENT_API_PORT
-
-  api_ssl = False
-  api_ssl_prop = properties.get_property(SSL_API)
-  if api_ssl_prop is not None:
-    api_ssl = api_ssl_prop.lower() == "true"
-
-  if api_ssl:
-    api_protocol = SERVER_API_SSL_PROTOCOL
-    api_port = DEFAULT_SSL_API_PORT
-    api_port_prop = properties.get_property(SSL_API_PORT)
-    if api_port_prop is not None:
-      api_port = api_port_prop
-
-  url = '{0}://{1}:{2!s}{3}'.format(api_protocol, SERVER_API_HOST, api_port, SERVER_API_LDAP_URL)
+  url = get_ambari_server_api_base(properties) + SERVER_API_LDAP_URL
   admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '')
   request = urllib2.Request(url)
   request.add_header('Authorization', 'Basic %s' % admin_auth)

ambari-server/src/main/python/upgradeHelper.py

@@ -165,7 +165,6 @@ class Options(Const):
   MODIFY_CONFIG_ACTION = "update-configs"
   BACKUP_CONFIG_ACTION = "backup-configs"
   INSTALL_YARN_MR2_ACTION = "install-yarn-mr2"
-  FINALIZE_RU_ACTION = "finalize-ru"
 
   MR_MAPPING_FILE = "mr_mapping"
   CAPACITY_SCHEDULER_TAG = "capacity-scheduler"
@@ -197,9 +196,6 @@ class Options(Const):
   # for verify action
   REPORT_FILE = None
 
-  # for finalize action
-  REPO_VERSION = None
-
   API_TOKENS = {
     "user": None,
     "pass": None
@@ -901,9 +897,7 @@ def install_services():
                          "the status of the install requests."
 
 
-def validate_response(response, expect_body, http_code):
-  if http_code is not None and http_code < 200 or http_code >= 300:
-    return 1, "HTTP code {0}\n".format(http_code) + response
+def validate_response(response, expect_body):
   if expect_body:
     if "\"href\" : \"" not in response:
       return 1, response
@@ -916,13 +910,16 @@ def validate_response(response, expect_body, http_code):
 
 
 def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=False,
-         simulate=None, validate=False, validate_expect_body=False, request_http_code=False):
+         simulate=None, validate=False, validate_expect_body=False):
 
   simulate_only = Options.CURL_PRINT_ONLY is not None or (simulate is not None and simulate is True)
   print_url = Options.CURL_PRINT_ONLY is not None and simulate is not None
 
   curl_path = '/usr/bin/curl'
-  curl_list = [curl_path, '-X', request_type]
+  curl_list = [curl_path]
+
+  curl_list.append('-X')
+  curl_list.append(request_type)
 
   if tokens is not None:
     curl_list.append('-u')
@@ -931,9 +928,6 @@ def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=Fa
     curl_list.append('-u')
     curl_list.append("%s:%s" % (Options.API_TOKENS["user"], Options.API_TOKENS["pass"]))
 
-  if request_http_code:
-    curl_list += ['-w', '\n%{http_code}']
-
   if request_type in Options.POST_REQUESTS:
     curl_list.append(url)
 
@@ -955,17 +949,12 @@ def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=Fa
   if print_url:
     Options.logger.info(" ".join(curl_list))
 
-  http_code = None
   if not simulate_only:
     osStat = subprocess.Popen(
       curl_list,
       stderr=subprocess.PIPE,
       stdout=subprocess.PIPE)
     out, err = osStat.communicate()
-    if request_http_code:
-      out_lines = out.splitlines()
-      http_code = int(out_lines[-1])
-      out = '\n'.join(out_lines[0:-1])
     if 0 != osStat.returncode:
       error = "curl call failed. out: " + out + " err: " + err
       Options.logger.error(error)
@@ -976,7 +965,7 @@ def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=Fa
     out = "{}"
 
   if validate and not simulate_only:
-    retcode, errdata = validate_response(out, validate_expect_body, http_code)
+    retcode, errdata = validate_response(out, validate_expect_body)
     if not retcode == 0:
       raise FatalException(retcode, errdata)
 
@@ -1145,18 +1134,6 @@ def verify_configuration():
       Options.logger.error("Report file close error: %s" % e.message)
 
 
-def finalize_ru():
-  TARGET_URL = Options.CLUSTER_URL + '/stack_versions'
-  request = {
-    "ClusterStackVersions": {
-       "repository_version": Options.REPO_VERSION,
-       "state": "CURRENT"
-    }
-  }
-  curl(TARGET_URL, request_type="PUT", data=request,
-       validate=True, validate_expect_body=False, request_http_code=True)
-
-
 def report_formatter(report_file, config_item, analyzed_list_item):
   prefix = "Configuration item %s" % config_item
   if analyzed_list_item["fail"]["count"] > 0:
@@ -1178,8 +1155,7 @@ def main():
                    Options.MODIFY_CONFIG_ACTION: modify_configs,
                    Options.INSTALL_YARN_MR2_ACTION: install_services,
                    Options.BACKUP_CONFIG_ACTION: backup_configs,
-                   Options.VERIFY_ACTION: verify_configuration,
-                   Options.FINALIZE_RU_ACTION: finalize_ru
+                   Options.VERIFY_ACTION: verify_configuration
   }
 
   parser = optparse.OptionParser(usage="usage: %prog [options] action\n  Valid actions: "
@@ -1203,8 +1179,6 @@ def main():
   parser.add_option('--password', default=None, help="Ambari admin password", dest="password")
   parser.add_option('--clustername', default=None, help="Cluster name", dest="clustername")
 
-  parser.add_option('--repository-version', default=None, help="Repository version", dest="repo_version")
-
   (options, args) = parser.parse_args()
   Options.initialize_logger(options.logfile)
   options.warnings = []
@@ -1237,10 +1211,6 @@ def main():
     if options.report is None:
       options.warnings.append("Should be provided report option")
 
-  if action == Options.FINALIZE_RU_ACTION:
-    if options.repo_version is None:
-      options.warnings.append("Should be provided repository-version option")
-
   if len(options.warnings) != 0:
     print parser.print_help()
     for warning in options.warnings:
@@ -1261,7 +1231,6 @@ def main():
     "pass": options.password
   }
   Options.REPORT_FILE = options.report
-  Options.REPO_VERSION = options.repo_version
 
   if action in action_list:
     Options.initialize()

ambari-server/src/test/python/TestUpgradeHelper.py

@@ -78,7 +78,6 @@ class TestUpgradeHelper(TestCase):
       report = "report.txt"
       warnings = []
       printonly = False
-      repo_version = None
 
     args = ["update-configs"]
     modify_action_mock.return_value = MagicMock()
@@ -94,57 +93,5 @@ class TestUpgradeHelper(TestCase):
     self.assertEqual(options.clustername, upgradeHelper.Options.CLUSTER_NAME)
 
 
-  @patch("optparse.OptionParser")
-  @patch("upgradeHelper.finalize_ru")
-  @patch("__builtin__.open")
-  def test_Finalize_options(self, open_mock, finalize_ru_mock, option_parser_mock):
-    class options(object):
-      user = "test_user"
-      hostname = "127.0.0.1"
-      clustername = "test1"
-      password = "test_password"
-      upgrade_json =  None
-      from_stack = None
-      to_stack = None
-      logfile = "test.log"
-      report = None
-      warnings = []
-      printonly = False
-      repo_version = None
-
-    args = ["finalize-ru"]
-    test_mock = MagicMock()
-    test_mock.parse_args = lambda: (options, args)
-    option_parser_mock.return_value = test_mock
-
-    try:
-      upgradeHelper.main()
-    except upgradeHelper.FatalException:
-      # Expected
-      pass
-
-    class options(object):
-      user = "test_user"
-      hostname = "127.0.0.1"
-      clustername = "test1"
-      password = "test_password"
-      upgrade_json =  None
-      from_stack = None
-      to_stack = None
-      logfile = "test.log"
-      report = None
-      warnings = []
-      printonly = False
-      repo_version = 'HDP-2.2.2.0-2561'
-
-    args = ["finalize-ru"]
-    test_mock = MagicMock()
-    test_mock.parse_args = lambda: (options, args)
-    option_parser_mock.return_value = test_mock
-
-    upgradeHelper.main()
-    self.assertTrue(finalize_ru_mock.called)
-
-
 if __name__ == "__main__":
   unittest.main()